IAP GITLAB

Skip to content
Snippets Groups Projects

Resolve "make friendly example scripts for the release"

Closed Alan Coleman requested to merge 626-make-friendly-example-scripts-for-the-release into master
2 unresolved threads
Compare and Show latest version
9 files
+ 81
23
Compare changes
  • Side-by-side
  • Inline
Files
9
@@ -8,8 +8,9 @@
the license.
"""
import pyarrow
import numpy as np
import pyarrow
def convert_to_numpy(pyarrow_table: pyarrow.lib.Table) -> np.ndarray:
"""
@@ -28,8 +29,8 @@ def convert_to_numpy(pyarrow_table: pyarrow.lib.Table) -> np.ndarray:
"""
# Type conversions for pyarrow data types to numpy ones
# pyarrow: https://arrow.apache.org/docs/python/data.html
# numpy: https://numpy.org/doc/stable/reference/arrays.dtypes.html#arrays-dtypes-constructing
# https://arrow.apache.org/docs/python/data.html
# https://numpy.org/doc/stable/reference/arrays.dtypes.html#arrays-dtypes-constructing
type_conversions = {
pyarrow.int8(): "int8",
pyarrow.int16(): "int16",
@@ -45,10 +46,13 @@ def convert_to_numpy(pyarrow_table: pyarrow.lib.Table) -> np.ndarray:
}
# Perform type conversion of all fields
column_types = [type_conversions[pyarrow_table[key].type] for key in pyarrow_table.column_names]
column_types = [
type_conversions[pyarrow_table[key].type] for key in pyarrow_table.column_names
]
dtypes = [(x, y) for (x, y) in zip(pyarrow_table.column_names, column_types)]
# Make an empty array and then fill the values
np_table = np.zeros(pyarrow_table.num_rows, dtype={"names": pyarrow_table.column_names, "formats": column_types})
np_table = np.zeros(pyarrow_table.num_rows, dtype=dtypes)
for key in pyarrow_table.column_names:
np_table[key] = pyarrow_table[key]
Loading