Apache Arrow Flight Data Stream

import pyarrow as pa
import pyarrow.ipc as ipc
import pandas as pd
from datetime import datetime

schema = pa.schema([
    ("id", pa.int64()),
    ("name", pa.string()),
    ("value", pa.float64()),
    ("timestamp", pa.timestamp("ms")),
    ("active", pa.bool_())
])

# Generate 10 batches of 500 rows (5000 rows)
#
number_of_batches=10
batch_size=500

with open("from_python.arrow", "wb") as f:
    #
    # Write data in an Arrow File Stream format.
    # This can be read back by Apache Hop using the Arrow File Stream data stream plugin type.
    #
    with ipc.new_stream(f, schema) as writer:
        for i in range(number_of_batches):
            batch = pa.RecordBatch.from_pydict({
                "id": list(range(i*batch_size+1, i*batch_size + batch_size+1)),
                "name": [f"row_{j}" for j in range(i*batch_size, i*batch_size + batch_size)],
                "value": [j * 1.23 for j in range(i*batch_size, i*batch_size + batch_size)],
                "timestamp": [datetime.now() for _ in range(batch_size)],
                "active": [j % 2 == 0 for j in range(i*batch_size, i*batch_size + batch_size)]
            }, schema=schema)
            writer.write_batch(batch)

print("✅ Wrote from_python.arrow in Arrow IPC Streaming Format")

import pyarrow as pa
import pyarrow.ipc as ipc

file_path = "/path/to/stream.arrow"

# Best practice: use memory mapping for performance
with pa.memory_map(file_path, 'r') as source:
    reader = ipc.open_stream(source)

    schema = reader.schema
    print("Schema:", schema)

    # Read everything into one Table (simple & efficient for most cases)
    table = pa.Table.from_batches(reader)     # or list(reader) then from_batches
    print(f"Total rows: {len(table)}")

    # Convert to pandas (or Polars) if needed
    df = table.to_pandas()

    df = df.reset_index()

    for index, row in df.iterrows():
        print(row['id'], row['str'], row['num'], row['int'], row['uuid'], row['sysdate'])

    #print(df.to_csv())