kjuulh/kafka-ingest-concept
1
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: add iceberg with spark

Browse Source 
Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
Kasper Juul Hermansen
2024-08-05 23:23:34 +02:00
parent a0be74c4d8
commit f345a956c9
6 changed files with 112 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
templates/spark.Dockerfile Normal file
View File

@@ -0,0 +1,16 @@
FROM bitnami/spark:3.3 as spark
USER root
RUN apt-get update && \
apt-get install -y curl && \
apt-get clean;
USER 1001
RUN rm -r /opt/bitnami/spark/jars && \
curl https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz | \
tar --extract --gzip --strip=1 --directory /opt/bitnami/spark/ spark-3.3.1-bin-hadoop3/jars/
RUN curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.231/aws-java-sdk-bundle-1.12.231.jar --output /opt/bitnami/spark/jars/aws-java-sdk-bundle-1.12.231.jar
RUN curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.1/hadoop-aws-3.3.1.jar --output /opt/bitnami/spark/jars/hadoop-aws-3.3.1.jar
RUN curl https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/0.9.4/jets3t-0.9.4.jar --output /opt/bitnami/spark/jars/jets3t-0.9.4.jar