feat: add basic ingest from spark iceberg example

This commit is contained in:
2025-01-31 23:59:21 +01:00
commit d27096896b
8 changed files with 460 additions and 0 deletions

20
spark/create_table.py Normal file
View File

@@ -0,0 +1,20 @@
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("").getOrCreate()
print("creating database")
spark.sql('CREATE DATABASE IF NOT EXISTS orders')
print("creating table")
spark.sql('''
CREATE TABLE IF NOT EXISTS orders.payments (
id STRING,
type STRING,
created_at TIMESTAMP,
document STRING,
payer STRING,
amount INT
)
USING iceberg
PARTITIONED BY (document)
''')