Apache Superset Query Data From Deltalake

1. Install minio client using pip.

2. Setup custom config at superset/docker/pythonpath_dev/superset_config.py

   def build_config(env_file=".env"):
       from dotenv import dotenv_values
    
       config = dotenv_values(env_file)
       if any(
           [
               config.get("AWS_ACCESS_KEY_ID") is None,
               config.get("AWS_SECRET_ACCESS_KEY") is None,
               config.get("AWS_ENDPOINT_URL") is None,
           ]
       ):
           raise ValueError("AWS credentials not found in .env file")
       return config
    
    
   def create_minio_client(env_file=".env"):
       """
       Creates a minio client using the credentials stored in the .env file.
       """
       from minio import Minio
    
       config = build_config(env_file=env_file)
    
       # remove http:// because Minio doesn't like it
       endpoint = config["AWS_ENDPOINT_URL"].split("//")[1]
       return Minio(
           endpoint,
           access_key=config["AWS_ACCESS_KEY_ID"],
           secret_key=config["AWS_SECRET_ACCESS_KEY"],
           secure=False,
       )
    
    
   def create_minio_file_url(bucket_name, table_path):
       from datetime import timedelta
       from deltalake import DeltaTable
    
       storage_options = {
           "AWS_ACCESS_KEY_ID":
           "AWS_SECRET_ACCESS_KEY":
           "AWS_ENDPOINT_URL":
           "AWS_REGION":
           "AWS_S3_ALLOW_UNSAFE_RENAME": "true",
           "AWS_STORAGE_ALLOW_HTTP": "true",
       }
       dt = DeltaTable(table_path, storage_options=storage_options)
    
       env_file = __file__.replace("superset_config.py", ".env")
       minioClient = create_minio_client(env_file=env_file)
    
       file_urls = []
    
       # Now you have a dictionary of object names and their respective download URLs
       for url in dt.file_uris():
           url = url.replace(f"s3://{bucket_name}/", "")
           file_urls.append(
               minioClient.presigned_get_object(bucket_name, url, expires=timedelta(days=1))
           )
       return file_urls
    
   JINJA_CONTEXT_ADDONS = {"delta_table": create_minio_file_url}

3. In Superset SQL-Lab.

   SELECT test_id, test_actual_duration from read_parquet({{delta_table('spatch', <table-path>)}}) LIMIT 7;