kjuulh/data-platform-in-a-box
1
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: add basic ingest from spark iceberg example

Browse Source
This commit is contained in:
Kasper Juul Hermansen
2025-01-31 23:59:21 +01:00
commit d27096896b
8 changed files with 460 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
notebooks/.ipynb_checkpoints/iceberg-checkpoint.ipynb Normal file
View File

@@ -0,0 +1,103 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "29f8d24e-e4bf-484d-afd4-cb82ff6cd50d",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SHOW DATABASES"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "70349765-e5f1-43a5-a141-cc2d54c69a58",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SHOW TABLES FROM orders"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fabaed9c-9049-4996-9d26-b20f66303911",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SHOW TBLPROPERTIES orders.payments"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6317d9c6-140e-4a63-890e-2173fbb9503e",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SELECT COUNT(*)\n",
"FROM orders.payments"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a1ff132-dc65-4943-a9be-416ba5a13c26",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SELECT *\n",
"FROM orders.payments\n",
"LIMIT 10"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2688a95-594c-45ad-9d49-70a1bcd59a1b",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SELECT * \n",
"FROM orders.payments.partitions\n",
"ORDER BY record_count DESC\n",
"LIMIT 10"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.17"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

103
notebooks/iceberg.ipynb Normal file
View File

@@ -0,0 +1,103 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "29f8d24e-e4bf-484d-afd4-cb82ff6cd50d",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SHOW DATABASES"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "70349765-e5f1-43a5-a141-cc2d54c69a58",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SHOW TABLES FROM orders"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fabaed9c-9049-4996-9d26-b20f66303911",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SHOW TBLPROPERTIES orders.payments"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6317d9c6-140e-4a63-890e-2173fbb9503e",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SELECT COUNT(*)\n",
"FROM orders.payments"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a1ff132-dc65-4943-a9be-416ba5a13c26",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SELECT *\n",
"FROM orders.payments\n",
"LIMIT 10"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2688a95-594c-45ad-9d49-70a1bcd59a1b",
"metadata": {},
"outputs": [],
"source": [
"%%sql\n",
"\n",
"SELECT * \n",
"FROM orders.payments.partitions\n",
"ORDER BY record_count DESC\n",
"LIMIT 10"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.17"
}
},
"nbformat": 4,
"nbformat_minor": 5
}