Upload 7 files

app.py

@@ -0,0 +1,422 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import networkx as nx
+import numpy as np
+import streamlit as st
+import sdv
+from sdv.datasets.local import load_csvs
+from sdv.metadata import MultiTableMetadata
+from sdv.multi_table import HMASynthesizer
+import time
+import os
+import gc
+import warnings
+from PIL import Image
+from sdv.metadata import SingleTableMetadata
+import pyodbc
+import google.generativeai as genai
+from google.generativeai.types import HarmCategory, HarmBlockThreshold
+import textwrap
+from streamlit_extras.stylable_container import stylable_container
+from streamlit_extras.stateful_button import button
+import json 
+from io import BytesIO
+genai.configure(api_key='AIzaSyCeY8jSHKW6t0OSDRjc2VAfBvMunVrff2w')
+genai_mod = genai.GenerativeModel(
+    model_name='models/gemini-pro'
+)
+
+st.set_page_config(page_title='DATA DISCOVERY', layout= 'wide')
+st.markdown("""
+    <style>
+    
+           /* Remove blank space at top and bottom */ 
+           .block-container {
+               padding-top: 2rem;
+            }
+           
+           /* Remove blank space at the center canvas */ 
+           .st-emotion-cache-z5fcl4 {
+               position: relative;
+               top: -62px;
+               }
+           
+           /* Make the toolbar transparent and the content below it clickable */ 
+           .st-emotion-cache-18ni7ap {
+               pointer-events: none;
+               background: rgb(255 255 255 / 0%)
+               }
+           .st-emotion-cache-zq5wmm {
+               pointer-events: auto;
+               background: rgb(255 255 255);
+               border-radius: 5px;
+               }
+    </style>
+    """, unsafe_allow_html=True)
+def clear_cache():
+    if 'rdf' in st.session_state:
+        st.session_state.pop('rdf')
+
+def create_er_diagram(df):
+    G = nx.DiGraph()  # Directed graph
+
+    # Dictionary to hold table columns
+    table_columns = {}
+
+    # Add nodes and edges to the graph
+    for _, row in df.iterrows():
+        parent_table = row['PARENT TABLE']
+        child_table = row['CHILD TABLE']
+        parent_pk = row['PARENT TABLE RELATIONSHIP COLUMN']
+        child_fk = row['CHILD TABLE RELATIONSHIP COLUMN']
+        cardinality = row.get('CARDINALITY', '1:N')
+
+        # Add columns to tables
+        if parent_table not in table_columns:
+            table_columns[parent_table] = []
+        table_columns[parent_table].append(parent_pk)
+
+        if child_table not in table_columns:
+            table_columns[child_table] = []
+        table_columns[child_table].append(child_fk)
+
+        # Add nodes and edges
+        G.add_node(parent_table)
+        G.add_node(child_table)
+        G.add_edge(parent_table, child_table, label=f'{parent_pk} -> {child_fk}\n{cardinality}')
+
+    return G, table_columns
+
+def draw_er_diagram(G, table_columns):
+    pos = nx.spring_layout(G, k=1.5, iterations=50)  # Use a layout that spreads out nodes
+
+    plt.figure(figsize=(8, 8))
+    nx.draw(G, pos, with_labels=False, node_size=2500, node_color='lightblue', edge_color='gray', font_size=8, font_weight='bold', arrows=True)
+
+    # Draw node labels (table names in bold)
+    for node, (x, y) in pos.items():
+        plt.text(x, y + 0.13, node, fontsize=7, fontweight='bold', ha='center', va='center')
+
+    # Draw column names
+    for node, columns in table_columns.items():
+        x, y = pos[node]
+        column_text = '\n'.join(columns)
+        plt.text(x, y, column_text, fontsize=6, ha='center', va='center')
+
+    # Draw edge labels
+    edge_labels = nx.get_edge_attributes(G, 'label')
+    nx.draw_networkx_edge_labels(G, pos, edge_labels=edge_labels, font_size=6)
+    st.subheader("Schematic Representation")
+    with st.container(border=True, height= 350):
+        st.pyplot(plt)
+    img_bytes = BytesIO()
+    plt.savefig(img_bytes, format='png')
+    img_bytes.seek(0)
+    return img_bytes
+
+def cardinality(parent_df, child_df, parent_column, child_column):
+    # Check uniqueness of parent primary key
+    is_parent_unique = parent_df[parent_column].is_unique
+
+    # Check uniqueness of child foreign key
+    is_child_unique = child_df[child_column].is_unique
+
+    # Determine cardinality
+    if is_parent_unique and is_child_unique:
+        return '1:1'
+    elif is_parent_unique and not is_child_unique:
+        return '1:N'
+    elif not is_parent_unique and is_child_unique:
+        return 'N:1'
+    else:
+        return 'N:N'
+
+#st.title('AUTOMATED DATA CATALOGUE')
+st.subheader('SELECT SOURCE')
+selectcol11, selectcol12 = st.columns(2)
+with selectcol11:
+    select1=st.selectbox('SOURCE DB NAME',('DB_10001','Marcopolo_db'),key='dbname',index=None,placeholder='Select database name', on_change=clear_cache)
+with selectcol12:
+    select2=st.selectbox('SOURCE SCHEMA NAME',('DBO','CLIENT'),key='SCHname',index=None,placeholder='Select schema name', on_change=clear_cache)
+if select1 =='DB_10001' and select2 is not None:
+    with st.spinner("Loading Tables:"):
+        conn1 = pyodbc.connect("Driver={ODBC Driver 17 for SQL Server};"
+                                        "Server=sql-ext-dev-uks-001.database.windows.net;"
+                                        "Database=sqldb-ext-dev-uks-001;"
+                                        "UID=dbadmin;"
+                                        "PWD=mYpa$$w0rD" )
+                    
+        query0_1=f"select * from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='{select2}' ORDER BY TABLE_NAME ASC"
+        st.session_state.tab_names_init=list(pd.read_sql_query(query0_1,con=conn1)['TABLE_NAME'])
+    
+    table_selector=st.multiselect('SOURCE TABLE NAME',st.session_state.tab_names_init,default=None,placeholder='Select table(s) for automated data cataloging', on_change= clear_cache)
+    sample_selector=st.selectbox('SELECT SAMPLE SIZE',['100','10K','100K','1M','Full Table'],index=None,placeholder='Select sample size for the table(s)', on_change= clear_cache)
+
+    discover= button("Discover", key='discover')
+
+    if discover:
+        if sample_selector=='100':
+            count="top 100"
+        elif sample_selector=='10K':
+            count="top 10000"
+        elif sample_selector=='100K':
+            count="top 100000"
+        elif sample_selector=='1M':
+            count="top 1000000"
+        else:
+            count=""
+        
+        query1_1=f"select * from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='{select2}' and TABLE_NAME in ("+(', '.join(f"'{table}'" for table in table_selector))+")  ORDER BY TABLE_NAME ASC"
+        st.session_state.tab_names=list(pd.read_sql_query(query1_1,con=conn1)['TABLE_NAME'])
+        st.session_state.dataframes = {}
+        st.session_state.col_names = []
+        for tab in st.session_state.tab_names:
+            query2_2= "select "+count+" * from ["+select2+"].["+tab+"]"
+            st.session_state.dataframes[f'{tab}'] = pd.read_sql_query(query2_2,con=conn1)
+            st.session_state.col_names = st.session_state.col_names + list(st.session_state.dataframes[f'{tab}'].columns)
+        #st.session_state.data_load = "Yes"
+
+        tab_names = st.session_state.tab_names
+        dataframes = st.session_state.dataframes
+        col_names = st.session_state.col_names
+        metadata = MultiTableMetadata()
+        metadata.detect_from_dataframes(
+            data= st.session_state.dataframes
+        )
+        multi_python_dict = metadata.to_dict()
+
+        st.markdown(f"System has ingested :orange[**{str(len(tab_names))} tables**] from the source. Please proceed with the discovery.")
+        #st.subheader("DATA CATALOGUE")
+        tab1, tab2= st.tabs(["Explain Tables", "Show Relationships"])
+        def view_callback():
+            st.session_state.tdet = False
+        with tab1:
+            #st.write(python_dict)
+            st.session_state.table_list= pd.DataFrame(tab_names,columns=['TABLE NAME'])
+            containter_length = (len(st.session_state.table_list) + 1)*35
+            tab_names_shown= list(st.session_state.table_list['TABLE NAME'].values)
+            tabs2= st.tabs(tab_names_shown)
+            for i, tab in enumerate(tabs2):
+                with tab:
+                    with st.container(height= 400, border=True):
+                        cole1,cole2=st.columns([1,1.5])
+                        with cole1:
+                            conn = pyodbc.connect("Driver={ODBC Driver 17 for SQL Server};"
+                                                                "Server=sql-ext-dev-uks-001.database.windows.net;"
+                                                                "Database=sqldb-ext-dev-uks-001;"
+                                                                "UID=dbadmin;"
+                                                                "PWD=mYpa$$w0rD" )
+                            table_selector= tab_names_shown[i]
+                            if table_selector is not None:
+                                query2="select "+count+" * from [dbo].["+table_selector+"]"
+                                #df = pd.read_sql_query(query2,con=conn)
+                                df = st.session_state.dataframes[table_selector]
+                                selected_df = pd.DataFrame()
+                                for col in df.columns:
+                                        # Filter non-null and non-blank values
+                                        non_null_values = df[col][df[col] != ''].dropna().astype(str).str.strip()
+                                        
+                                        # Select up to 10 values (or fewer if less than 10 non-null values)
+                                        selected_values = list(non_null_values[:10])
+                                        selected_values = selected_values + [""] * (10 - len(selected_values))
+                                        # Add selected values to the new dataframe
+                                        selected_df[col] = selected_values
+                                #st.dataframe(selected_df)
+                                null_columns = [col for col in selected_df.columns if selected_df.apply(lambda x: x == '')[col].nunique() > 1]
+                                null_mes= "**The Following columns have very few records(less than 10). You might exclude them (if they are redundant) for better table discovery:** \n\n"
+                                for col in null_columns[:-1]:
+                                    null_mes += f":orange[**{col}**]" + ', '
+                                for collast in null_columns[-1:]:
+                                    if len(null_columns)> 1:
+                                        null_mes += '**and** ' + f":orange[**{collast}**]"
+                                    else: 
+                                        null_mes += f":orange[**{collast}**]" 
+                                        
+                                if len(null_columns) != 0:
+                                    with st.expander("🛈 Potential redundant Columns Found in Terms of Data Completeness:", expanded= True):
+                                        st.markdown(null_mes)
+                                    inf_filter= st.multiselect('Select Incomplete and Insignificant Columns to exclude:', list(null_columns))
+                                    run = st.button('Check', key= f"{tab_names_shown[i]}")
+                                else:
+                                    st.success("No redundant Columns Found in Terms of Data Completeness")
+                                    inf_filter= None
+                                    run = False
+                                
+                                if inf_filter is not None:
+                                    df.drop(columns=inf_filter, inplace=True)
+                                    selected_df.drop(columns=inf_filter, inplace=True)
+                
+                                if run or len(null_columns) == 0:
+                                    main_list=df.columns.to_list()
+                                    sub_list=['ID','LOADID','FILE_NAME']
+                                    if any(main_list[i:i+len(sub_list)] == sub_list for i in range(len(main_list) - len(sub_list) + 1)):
+                                        df=df.drop(['ID','LOADID','FILE_NAME'],axis=1)
+                                    conn.close()
+                                    sin_metadata = SingleTableMetadata()
+                                    sin_metadata.detect_from_dataframe(df)
+                                    python_dict = sin_metadata.to_dict()
+                                    if f'cont_{table_selector}' not in st.session_state:
+                                        with st.spinner("Processing Table"):
+                                            # Create a GenerativeModel instance
+                                            genai_mod = genai.GenerativeModel(
+                                                model_name='models/gemini-pro'
+                                            )
+                                            if 'primary_key' in python_dict:
+                                                primary_key = python_dict['primary_key']
+                                            else:
+                                                primary_key = "Could Not be Identified"
+                                            
+                                            
+                                            story =  f""" Details of the table: 
+                                            table columns: {str(list(df.columns))}
+                                            column datatypes: {str(df.dtypes.to_string())}
+                                            table sample data: {selected_df.head(10).to_string()}
+                                            """
+                                            response = genai_mod.generate_content(textwrap.dedent("""
+                                                    You are a Data Migration expert. You can analyze and understand any table/data/ Please return a narration about the data. The narration should Include primary key name(if any) and a intellectual guess about the table schema. The data can be any kind of generic data. you have to guess the object name/class name/schema name etc. of that data. Don't add unnecessary details. Strictly stick to the informations provided only.
+                            Important: Please consider All fields are mandetorily during your analysis. Explain all fields precisely without unnecessary and irrelevant information. NO NEED TO PROVIDE THE SAMPLE DATA AGAIN.
+                                                
+                                                        Here is the table details:
+                                                
+                                                        """) + story + f"The Primary Key is:{primary_key}" ,
+                                                        safety_settings={
+                                                            HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
+                                                            HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
+                                                            HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
+                                                            HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
+                                                        })
+                                            st.session_state[f'cont_{table_selector}'] = response.text
+    
+                                    st.markdown(st.session_state[f'cont_{table_selector}'])
+                        with cole2:
+                            st.markdown("**DATA PREVIEW**")
+                            st.dataframe(df, use_container_width= True)
+                            
+        with tab2:
+            metadata1 = MultiTableMetadata()
+            metadata1.detect_from_dataframes(
+                data= st.session_state.dataframes
+            )
+            multi_python_dict1 = metadata1.to_dict()
+            rlist1=multi_python_dict1['relationships']
+            rdf=pd.DataFrame(columns=['PARENT TABLE','CHILD TABLE','PARENT TABLE RELATIONSHIP COLUMN','CHILD TABLE RELATIONSHIP COLUMN','CARDINALITY'])
+            for i in range(len(rlist1)):
+                rlist=rlist1[i]
+                nrow=pd.DataFrame({'PARENT TABLE':rlist['parent_table_name'],'CHILD TABLE':rlist['child_table_name'],'PARENT TABLE RELATIONSHIP COLUMN':rlist['parent_primary_key'],'CHILD TABLE RELATIONSHIP COLUMN':rlist['child_foreign_key']},index=[i])
+                rdf=pd.concat([rdf,nrow],ignore_index=True)
+
+            rdf['CARDINALITY'] = rdf.apply(
+                lambda row: cardinality(
+                    st.session_state.dataframes[str(row['PARENT TABLE'])],
+                        st.session_state.dataframes[str(row['CHILD TABLE'])],
+                            str(row['PARENT TABLE RELATIONSHIP COLUMN']),
+                                str(row['CHILD TABLE RELATIONSHIP COLUMN'])),axis=1)
+            
+            
+            if 'rdf' not in st.session_state:
+                st.session_state.rdf = rdf
+
+            edited_map_df =  st.data_editor(
+                                            st.session_state.rdf,
+                                            column_config={
+                                                "PARENT TABLE": st.column_config.SelectboxColumn(
+                                                    "Available Parent Table",
+                                                    width="medium",
+                                                    options=tab_names,
+                                                    required=True,
+                                                ),
+                                                "CHILD TABLE": st.column_config.SelectboxColumn(
+                                                    "Available Child Table",
+                                                    width="medium",
+                                                    options=tab_names,
+                                                    required=True,
+                                                ),
+                                                "PARENT TABLE RELATIONSHIP COLUMN": st.column_config.SelectboxColumn(
+                                                    "Available Parent Table Relationship Column",
+                                                    width="medium",
+                                                    options=col_names,
+                                                    required=True,
+                                                ),
+                                                "CHILD TABLE RELATIONSHIP COLUMN": st.column_config.SelectboxColumn(
+                                                    "Available Child Table Relationship Column",
+                                                    width="medium",
+                                                    options=col_names,
+                                                    required=True,
+                                                ),
+                                                "CARDINALITY": st.column_config.SelectboxColumn(
+                                                    "Cardinality",
+                                                    width="medium",
+                                                    options=['1:1','1:N','N:1','N:N'],
+                                                    required=True,
+                                                )
+                                            },
+                                            hide_index=True,
+                                            num_rows = 'dynamic',
+                                            use_container_width = True
+                                        )
+
+            for i,row in edited_map_df.iterrows():
+                pcolchecklist = st.session_state.dataframes[str(row['PARENT TABLE'])].columns
+                ccolchecklist = st.session_state.dataframes[str(row['CHILD TABLE'])].columns
+                pvals= list(st.session_state.dataframes[str(row['PARENT TABLE'])][row['PARENT TABLE RELATIONSHIP COLUMN']].values)
+                cvals= list(st.session_state.dataframes[str(row['CHILD TABLE'])][row['CHILD TABLE RELATIONSHIP COLUMN']].values)
+                match = [val for val in pvals if val in cvals]
+                #st.write(match)
+                if row['PARENT TABLE RELATIONSHIP COLUMN'] not in pcolchecklist:
+                    st.error(f"{row['PARENT TABLE RELATIONSHIP COLUMN']} does not belong to {row['PARENT TABLE']}")
+                else:
+                    pass
+                if row['CHILD TABLE RELATIONSHIP COLUMN'] not in ccolchecklist:
+                    st.error(f"{row['CHILD TABLE RELATIONSHIP COLUMN']} does not belong to {row['CHILD TABLE']}")
+                else:
+                    pass
+                if (row['PARENT TABLE RELATIONSHIP COLUMN'] in pcolchecklist) and (row['CHILD TABLE RELATIONSHIP COLUMN'] in ccolchecklist):
+                    pvals= list(st.session_state.dataframes[str(row['PARENT TABLE'])][row['PARENT TABLE RELATIONSHIP COLUMN']].values)
+                    cvals= list(st.session_state.dataframes[str(row['CHILD TABLE'])][row['CHILD TABLE RELATIONSHIP COLUMN']].values)
+                    match = [val for val in pvals if val in cvals]
+                    if match == []:
+                        st.error(f"The Joining Condition Between column: {row['PARENT TABLE RELATIONSHIP COLUMN']} from Table: {row['PARENT TABLE']} and column: {row['CHILD TABLE RELATIONSHIP COLUMN']} from Table: {row['CHILD TABLE']} does not yield any record. ")
+                if ((row['PARENT TABLE RELATIONSHIP COLUMN'] in pcolchecklist) and (row['CHILD TABLE RELATIONSHIP COLUMN'] in ccolchecklist)) and (match != []):
+                    # primary_check = len(list(dataframes[str(row['PARENT TABLE'])][row['PARENT TABLE RELATIONSHIP COLUMN']].values)) == dataframes[str(row['PARENT TABLE'])][row['PARENT TABLE RELATIONSHIP COLUMN']].nunique()
+                    # if primary_check:
+                    #     pass
+                    # else:
+                    #     st.error(f"The Column {row['PARENT TABLE RELATIONSHIP COLUMN']} from Table: {row['PARENT TABLE']} has duplicate records and hence can not be considered as Primary Key.")
+                    pass
+
+            add = st.button("Add Relationship", key='add')
+            if add:
+                if ((row['PARENT TABLE RELATIONSHIP COLUMN'] in pcolchecklist) and (row['CHILD TABLE RELATIONSHIP COLUMN'] in ccolchecklist)) and ((match != [])):
+                    add_df = edited_map_df
+                else:
+                    add_df = st.session_state.rdf            
+            else:
+                add_df = st.session_state.rdf
+            
+            add_df['CARDINALITY'] = add_df.apply(
+                lambda row: cardinality(
+                    st.session_state.dataframes[str(row['PARENT TABLE'])],
+                        st.session_state.dataframes[str(row['CHILD TABLE'])],
+                            str(row['PARENT TABLE RELATIONSHIP COLUMN']),
+                                str(row['CHILD TABLE RELATIONSHIP COLUMN'])),axis=1)
+                                
+            st.session_state.add_df = add_df
+            edited_map_df = st.session_state.add_df
+
+            rel_tabs = list(add_df['PARENT TABLE'].values) + list(add_df['CHILD TABLE'].values)
+            unrel_tabs = [tab for tab in tab_names if tab not in rel_tabs]
+            st.info(f"""Unrelated tables due to undetected pattern: {str(unrel_tabs).replace("[","").replace("]","")}""")
+
+            G, table_columns = create_er_diagram(st.session_state.add_df)
+            img_bytes= draw_er_diagram(G, table_columns)
+            col21, col22= st.columns([1,8])
+            with col21:
+                if st.button("Regenerate"):
+                    st.rerun()
+            with col22:
+                st.download_button(
+                    label="Download ER Diagram",
+                    data=img_bytes,
+                    file_name="er_diagram.png",
+                    mime="image/png"
+                )

lottie_2.json

@@ -0,0 +1 @@
+{"v":"5.4.1","fr":29.9700012207031,"ip":0,"op":132.00000537647,"w":1024,"h":768,"nm":"Comp 1","ddd":0,"assets":[],"layers":[{"ddd":0,"ind":1,"ty":3,"nm":"Null 1","sr":1,"ks":{"o":{"a":0,"k":0,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[512,364,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"ip":0,"op":316.000012870944,"st":0,"bm":0,"completed":true},{"ddd":0,"ind":2,"ty":4,"nm":"Shape Layer 3","parent":4,"sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[15.069,93,0],"ix":2},"a":{"a":0,"k":[0,93,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.833,"y":0.833},"o":{"x":0.167,"y":0.167},"n":"0p833_0p833_0p167_0p167","t":30,"s":[{"i":[[101.919,-93],[119.5,-85.113],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-119.5,-40.823],[-109.55199999999999,-63],[25.468,-63],[34.5,-67.05199999999999],[34.5,-75.419],[42.387,-93]],"o":[[111.613,-93],[119.5,-75.419],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-119.5,-53.052],[-97.323,-63],[30.448,-63],[34.5,-72.032],[34.5,-85.113],[52.081,-93]],"v":[[101.919,-93],[119.5,-75.419],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-119.5,-40.823],[-97.323,-63],[25.468,-63],[34.5,-72.032],[34.5,-75.419],[52.081,-93]],"c":true}],"e":[{"i":[[135.824,-47.794],[151.816,-39.604],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-89.363,4.383],[-77.907,-17.794],[55.605,-17.794],[64.637,-21.846],[66.333,-30.213],[76.291,-47.794]],"o":[[145.518,-47.794],[149.637,-30.213],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-85.138,-8.337],[-65.678,-17.794],[60.584999999999994,-17.794],[64.637,-26.826],[69.505,-41.11],[85.985,-47.794]],"v":[[135.824,-47.794],[149.637,-30.213],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-89.363,4.383],[-65.678,-17.794],[55.605,-17.794],[64.637,-26.826],[66.333,-30.213],[85.985,-47.794]],"c":true}]},{"i":{"x":0.833,"y":0.833},"o":{"x":0.167,"y":0.167},"n":"0p833_0p833_0p167_0p167","t":35,"s":[{"i":[[135.824,-47.794],[151.816,-39.604],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-89.363,4.383],[-77.907,-17.794],[55.605,-17.794],[64.637,-21.846],[66.333,-30.213],[76.291,-47.794]],"o":[[145.518,-47.794],[149.637,-30.213],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-85.138,-8.337],[-65.678,-17.794],[60.584999999999994,-17.794],[64.637,-26.826],[69.505,-41.11],[85.985,-47.794]],"v":[[135.824,-47.794],[149.637,-30.213],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-89.363,4.383],[-65.678,-17.794],[55.605,-17.794],[64.637,-26.826],[66.333,-30.213],[85.985,-47.794]],"c":true}],"e":[{"i":[[121.293,-67.168],[137.966,-59.108000000000004],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-102.279,-14.991],[-91.469,-37.168],[42.689,-37.168],[51.721,-41.22],[52.69,-49.587],[61.760999999999996,-67.168]],"o":[[130.987,-67.168],[136.721,-49.587],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-99.865,-27.5],[-79.24,-37.168],[47.669,-37.168],[51.721,-46.2],[54.501999999999995,-59.969],[71.455,-67.168]],"v":[[121.293,-67.168],[136.721,-49.587],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-102.279,-14.991],[-79.24,-37.168],[42.689,-37.168],[51.721,-46.2],[52.69,-49.587],[71.455,-67.168]],"c":true}]},{"i":{"x":0.833,"y":0.833},"o":{"x":0.167,"y":0.167},"n":"0p833_0p833_0p167_0p167","t":40,"s":[{"i":[[121.293,-67.168],[137.966,-59.108000000000004],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-102.279,-14.991],[-91.469,-37.168],[42.689,-37.168],[51.721,-41.22],[52.69,-49.587],[61.760999999999996,-67.168]],"o":[[130.987,-67.168],[136.721,-49.587],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-99.865,-27.5],[-79.24,-37.168],[47.669,-37.168],[51.721,-46.2],[54.501999999999995,-59.969],[71.455,-67.168]],"v":[[121.293,-67.168],[136.721,-49.587],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-102.279,-14.991],[-79.24,-37.168],[42.689,-37.168],[51.721,-46.2],[52.69,-49.587],[71.455,-67.168]],"c":true}],"e":[{"i":[[135.824,-47.794],[151.816,-39.604],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-89.363,4.383],[-77.907,-17.794],[55.605,-17.794],[64.637,-21.846],[66.333,-30.213],[76.291,-47.794]],"o":[[145.518,-47.794],[149.637,-30.213],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-85.138,-8.337],[-65.678,-17.794],[60.584999999999994,-17.794],[64.637,-26.826],[69.505,-41.11],[85.985,-47.794]],"v":[[135.824,-47.794],[149.637,-30.213],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-89.363,4.383],[-65.678,-17.794],[55.605,-17.794],[64.637,-26.826],[66.333,-30.213],[85.985,-47.794]],"c":true}]},{"i":{"x":0.833,"y":0.833},"o":{"x":0.167,"y":0.167},"n":"0p833_0p833_0p167_0p167","t":45,"s":[{"i":[[135.824,-47.794],[151.816,-39.604],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-89.363,4.383],[-77.907,-17.794],[55.605,-17.794],[64.637,-21.846],[66.333,-30.213],[76.291,-47.794]],"o":[[145.518,-47.794],[149.637,-30.213],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-85.138,-8.337],[-65.678,-17.794],[60.584999999999994,-17.794],[64.637,-26.826],[69.505,-41.11],[85.985,-47.794]],"v":[[135.824,-47.794],[149.637,-30.213],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-89.363,4.383],[-65.678,-17.794],[55.605,-17.794],[64.637,-26.826],[66.333,-30.213],[85.985,-47.794]],"c":true}],"e":[{"i":[[135.824,-47.794],[151.816,-39.604],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-89.363,4.383],[-77.907,-17.794],[55.605,-17.794],[64.637,-21.846],[66.333,-30.213],[76.291,-47.794]],"o":[[145.518,-47.794],[149.637,-30.213],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-85.138,-8.337],[-65.678,-17.794],[60.584999999999994,-17.794],[64.637,-26.826],[69.505,-41.11],[85.985,-47.794]],"v":[[135.824,-47.794],[149.637,-30.213],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-89.363,4.383],[-65.678,-17.794],[55.605,-17.794],[64.637,-26.826],[66.333,-30.213],[85.985,-47.794]],"c":true}]},{"i":{"x":0.833,"y":0.833},"o":{"x":0.167,"y":0.167},"n":"0p833_0p833_0p167_0p167","t":124,"s":[{"i":[[135.824,-47.794],[151.816,-39.604],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-89.363,4.383],[-77.907,-17.794],[55.605,-17.794],[64.637,-21.846],[66.333,-30.213],[76.291,-47.794]],"o":[[145.518,-47.794],[149.637,-30.213],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-85.138,-8.337],[-65.678,-17.794],[60.584999999999994,-17.794],[64.637,-26.826],[69.505,-41.11],[85.985,-47.794]],"v":[[135.824,-47.794],[149.637,-30.213],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-89.363,4.383],[-65.678,-17.794],[55.605,-17.794],[64.637,-26.826],[66.333,-30.213],[85.985,-47.794]],"c":true}],"e":[{"i":[[101.919,-93],[119.5,-85.113],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-119.5,-40.823],[-109.55199999999999,-63],[25.468,-63],[34.5,-67.05199999999999],[34.5,-75.419],[42.387,-93]],"o":[[111.613,-93],[119.5,-75.419],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-119.5,-53.052],[-97.323,-63],[30.448,-63],[34.5,-72.032],[34.5,-85.113],[52.081,-93]],"v":[[101.919,-93],[119.5,-75.419],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-119.5,-40.823],[-97.323,-63],[25.468,-63],[34.5,-72.032],[34.5,-75.419],[52.081,-93]],"c":true}]},{"t":128.000005213547}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false,"_render":true},{"ty":"st","c":{"a":0,"k":[0.5,0.5,0.5,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":6,"ix":5},"lc":1,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":8},"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false,"_render":true},{"ty":"fl","c":{"a":0,"k":[0.5,0.5,0.5,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false,"_render":true},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform","_render":true}],"nm":"Shape 1","np":3,"cix":2,"ix":1,"mn":"ADBE Vector Group","hd":false,"_render":true}],"ip":0,"op":316.000012870944,"st":0,"bm":0,"completed":true},{"ddd":0,"ind":3,"ty":4,"nm":"Shape Layer 4","parent":1,"sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[0,0,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[128.5,-143],[160.291,-218.596],[185,-151],[179.5,-190.5]],"o":[[128.5,-143],[214,-176.5],[159.481,-178.482],[188.742,-168.549]],"v":[[128.5,-143],[195.5,-191],[172,-165],[183.5,-181]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false,"_render":true},{"ty":"gs","o":{"a":0,"k":100,"ix":9},"w":{"a":0,"k":6,"ix":10},"g":{"p":3,"k":{"a":0,"k":[0,0.5,0.5,0.5,0.5,0.46,0.46,0.46,1,0.5,0.5,0.5],"ix":8}},"s":{"a":0,"k":[0,0],"ix":4},"e":{"a":0,"k":[100,0],"ix":5},"t":1,"lc":2,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":13},"nm":"Gradient Stroke 1","mn":"ADBE Vector Graphic - G-Stroke","hd":false,"_render":true},{"ty":"fl","c":{"a":0,"k":[0.5,0.5,0.5,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":true,"_render":true},{"ty":"tr","p":{"a":0,"k":[10,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[-67.81,84.615],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform","_render":true}],"nm":"Shape 5","np":3,"cix":2,"ix":1,"mn":"ADBE Vector Group","hd":false,"_render":true},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[128.5,-143],[160.291,-218.596],[185,-151],[179.5,-190.5]],"o":[[128.5,-143],[214,-176.5],[159.481,-178.482],[188.742,-168.549]],"v":[[128.5,-143],[195.5,-191],[172,-165],[183.5,-181]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false,"_render":true},{"ty":"fl","c":{"a":0,"k":[0.5,0.5,0.5,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":true,"_render":true},{"ty":"gs","o":{"a":0,"k":100,"ix":9},"w":{"a":0,"k":6,"ix":10},"g":{"p":3,"k":{"a":0,"k":[0,0.5,0.5,0.5,0.5,0.46,0.46,0.46,1,0.5,0.5,0.5],"ix":8}},"s":{"a":0,"k":[0,0],"ix":4},"e":{"a":0,"k":[100,0],"ix":5},"t":1,"lc":2,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":13},"nm":"Gradient Stroke 1","mn":"ADBE Vector Graphic - G-Stroke","hd":false,"_render":true},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform","_render":true}],"nm":"Shape 4","np":3,"cix":2,"ix":2,"mn":"ADBE Vector Group","hd":false,"_render":true},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[20,-36],[18,-162]],"o":[[20,-36],[82,-234]],"v":[[20,-36],[50,-198]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false,"_render":true},{"ty":"fl","c":{"a":0,"k":[0.5,0.5,0.5,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":true,"_render":true},{"ty":"gs","o":{"a":0,"k":100,"ix":9},"w":{"a":0,"k":6,"ix":10},"g":{"p":3,"k":{"a":0,"k":[0,0.5,0.5,0.5,0.5,0.46,0.46,0.46,1,0.5,0.5,0.5],"ix":8}},"s":{"a":0,"k":[0,0],"ix":4},"e":{"a":0,"k":[100,0],"ix":5},"t":1,"lc":2,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":13},"nm":"Gradient Stroke 1","mn":"ADBE Vector Graphic - G-Stroke","hd":false,"_render":true},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform","_render":true}],"nm":"Shape 3","np":3,"cix":2,"ix":3,"mn":"ADBE Vector Group","hd":false,"_render":true},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[-74,-30],[-131,-133]],"o":[[-74,-30],[-285,-99]],"v":[[-74,-30],[-208,-116]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false,"_render":true},{"ty":"fl","c":{"a":0,"k":[0.5,0.5,0.5,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":true,"_render":true},{"ty":"gs","o":{"a":0,"k":100,"ix":9},"w":{"a":0,"k":6,"ix":10},"g":{"p":3,"k":{"a":0,"k":[0,0.5,0.5,0.5,0.5,0.46,0.46,0.46,1,0.5,0.5,0.5],"ix":8}},"s":{"a":0,"k":[0,0],"ix":4},"e":{"a":0,"k":[100,0],"ix":5},"t":1,"lc":2,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":13},"nm":"Gradient Stroke 1","mn":"ADBE Vector Graphic - G-Stroke","hd":false,"_render":true},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform","_render":true}],"nm":"Shape 2","np":3,"cix":2,"ix":4,"mn":"ADBE Vector Group","hd":false,"_render":true},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[129,-67],[223,-164]],"o":[[129,-67],[393,-96]],"v":[[129,-67],[308,-130]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false,"_render":true},{"ty":"fl","c":{"a":0,"k":[0.5,0.5,0.5,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":true,"_render":true},{"ty":"gs","o":{"a":0,"k":100,"ix":9},"w":{"a":0,"k":6,"ix":10},"g":{"p":3,"k":{"a":0,"k":[0,0.5,0.5,0.5,0.5,0.46,0.46,0.46,1,0.5,0.5,0.5],"ix":8}},"s":{"a":0,"k":[0,0],"ix":4},"e":{"a":0,"k":[100,0],"ix":5},"t":1,"lc":2,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":13},"nm":"Gradient Stroke 1","mn":"ADBE Vector Graphic - G-Stroke","hd":false,"_render":true},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform","_render":true}],"nm":"Shape 1","np":3,"cix":2,"ix":5,"mn":"ADBE Vector Group","hd":false,"_render":true},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0.167]},"n":["0_1_0p167_0p167"],"t":40,"s":[0],"e":[100]},{"t":58.0000023623884}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0.167]},"n":["0_1_0p167_0p167"],"t":36,"s":[0],"e":[100]},{"t":54.0000021994651}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":6,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false,"_render":true}],"ip":0,"op":316.000012870944,"st":0,"bm":0,"completed":true},{"ddd":0,"ind":4,"ty":4,"nm":"Shape Layer 5","parent":1,"sr":1,"ks":{"o":{"a":0,"k":50,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[-13,138,0],"ix":2},"a":{"a":0,"k":[0,93,0],"ix":1},"s":{"a":1,"k":[{"i":{"x":[0.924,0.924,0.667],"y":[1,1,1]},"o":{"x":[0.751,0.751,0.333],"y":[0,0,0]},"n":["0p924_1_0p751_0","0p924_1_0p751_0","0p667_1_0p333_0"],"t":16,"s":[49.726,49.726,100],"e":[132.726,132.726,100]},{"i":{"x":[0.924,0.924,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"n":["0p924_1_0p333_0","0p924_1_0p333_0","0p667_1_0p333_0"],"t":30,"s":[132.726,132.726,100],"e":[132.726,132.726,100]},{"i":{"x":[0.833,0.833,0.833],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"n":["0p833_1_0p333_0","0p833_1_0p333_0","0p833_1_0p333_0"],"t":118,"s":[132.726,132.726,100],"e":[49.726,49.726,100]},{"t":128.000005213547}],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.833,"y":0.833},"o":{"x":0.167,"y":0.167},"n":"0p833_0p833_0p167_0p167","t":32,"s":[{"i":[[101.919,-93],[119.5,-85.113],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-119.5,-40.823],[-109.55199999999999,-63],[25.468,-63],[34.5,-67.05199999999999],[34.5,-75.419],[42.387,-93]],"o":[[111.613,-93],[119.5,-75.419],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-119.5,-53.052],[-97.323,-63],[30.448,-63],[34.5,-72.032],[34.5,-85.113],[52.081,-93]],"v":[[101.919,-93],[119.5,-75.419],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-119.5,-40.823],[-97.323,-63],[25.468,-63],[34.5,-72.032],[34.5,-75.419],[52.081,-93]],"c":true}],"e":[{"i":[[109.669,-82.667],[126.887,-74.712],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-112.611,-30.49],[-102.319,-52.667],[32.356,-52.667],[41.389,-56.72],[41.776,-65.087],[50.135999999999996,-82.667]],"o":[[119.363,-82.667],[126.389,-65.087],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-111.64500000000001,-42.830999999999996],[-90.09,-52.667],[37.336,-52.667],[41.389,-61.7],[42.501000000000005,-75.056],[59.83,-82.667]],"v":[[109.669,-82.667],[126.389,-65.087],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-112.611,-30.49],[-90.09,-52.667],[32.356,-52.667],[41.389,-61.7],[41.776,-65.087],[59.83,-82.667]],"c":true}]},{"i":{"x":0.833,"y":0.833},"o":{"x":0.167,"y":0.167},"n":"0p833_0p833_0p167_0p167","t":36,"s":[{"i":[[109.669,-82.667],[126.887,-74.712],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-112.611,-30.49],[-102.319,-52.667],[32.356,-52.667],[41.389,-56.72],[41.776,-65.087],[50.135999999999996,-82.667]],"o":[[119.363,-82.667],[126.389,-65.087],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-111.64500000000001,-42.830999999999996],[-90.09,-52.667],[37.336,-52.667],[41.389,-61.7],[42.501000000000005,-75.056],[59.83,-82.667]],"v":[[109.669,-82.667],[126.389,-65.087],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-112.611,-30.49],[-90.09,-52.667],[32.356,-52.667],[41.389,-61.7],[41.776,-65.087],[59.83,-82.667]],"c":true}],"e":[{"i":[[101.919,-93],[119.5,-85.113],[119.5,70.823],[109.55199999999999,93],[-97.323,93],[-119.5,83.05199999999999],[-119.5,-40.823],[-109.55199999999999,-63],[25.468,-63],[34.5,-67.05199999999999],[34.5,-75.419],[42.387,-93]],"o":[[111.613,-93],[119.5,-75.419],[119.5,83.05199999999999],[97.323,93],[-109.55199999999999,93],[-119.5,70.823],[-119.5,-53.052],[-97.323,-63],[30.448,-63],[34.5,-72.032],[34.5,-85.113],[52.081,-93]],"v":[[101.919,-93],[119.5,-75.419],[119.5,70.823],[97.323,93],[-97.323,93],[-119.5,70.823],[-119.5,-40.823],[-97.323,-63],[25.468,-63],[34.5,-72.032],[34.5,-75.419],[52.081,-93]],"c":true}]},{"t":40.0000016292334}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false,"_render":true},{"ty":"st","c":{"a":0,"k":[1,1,1,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":6,"ix":5},"lc":1,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":8},"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false,"_render":true},{"ty":"fl","c":{"a":0,"k":[0.5,0.5,0.5,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":true,"_render":true},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform","_render":true}],"nm":"Shape 1","np":3,"cix":2,"ix":1,"mn":"ADBE Vector Group","hd":false,"_render":true}],"ip":0,"op":316.000012870944,"st":0,"bm":0,"completed":true},{"ddd":0,"ind":5,"ty":4,"nm":"NO DATA Outlines","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"s":true,"x":{"a":0,"k":452,"ix":3},"y":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.03],"y":[0.238]},"n":["0p667_1_0p03_0p238"],"t":35,"s":[344],"e":[204]},{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"n":["0p667_1_0p333_0"],"t":45,"s":[204],"e":[204]},{"i":{"x":[0.957],"y":[0.75]},"o":{"x":[0.333],"y":[0]},"n":["0p957_0p75_0p333_0"],"t":98,"s":[204],"e":[324]},{"t":105.000004276738}],"ix":4}},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[122.539,122.539,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[11.813,-28.459],[11.813,-15.645],[7.664,-28.459],[1.477,-28.459],[1.477,0],[7.664,0],[7.664,-12.938],[11.514,0],[18,0],[18,-28.459]],"o":[[11.813,-28.459],[11.813,-15.645],[7.664,-28.459],[1.477,-28.459],[1.477,0],[7.664,0],[7.664,-12.938],[11.514,0],[18,0],[18,-28.459]],"v":[[11.813,-28.459],[11.813,-15.645],[7.664,-28.459],[1.477,-28.459],[1.477,0],[7.664,0],[7.664,-12.938],[11.514,0],[18,0],[18,-28.459]],"c":true},"ix":2},"nm":"N","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":2,"ix":5},"lc":1,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":8},"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":true},{"ty":"fl","c":{"a":0,"k":[0.92,0.92,0.92,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"N","np":3,"cix":2,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[37.881,-16.664],[37.811,-21.644],[37.089999999999996,-25.072000000000003],[34.638,-27.811999999999998],[30.843,-29.057],[26.309,-28.793],[22.937,-26.982],[21.097,-23.926],[20.76,-19.523000000000003],[20.76,-11.795],[20.830000000000002,-6.8149999999999995],[21.551000000000002,-3.387],[24.003,-0.647],[27.796,0.598],[32.332,0.334],[35.70399999999999,-1.4769999999999999],[37.544000000000004,-4.532],[37.881,-8.936]],"o":[[37.881,-19.594],[37.529,-23.988],[35.614,-27.064],[32.211999999999996,-28.808],[27.714,-29.057],[23.895,-27.738999999999997],[21.519000000000002,-25.014000000000003],[20.826999999999998,-21.546999999999997],[20.76,-16.664],[20.76,-8.865],[21.112,-4.471],[23.027,-1.395],[26.429000000000002,0.349],[30.925,0.598],[34.746,-0.72],[37.122,-3.445],[37.814,-6.9110000000000005],[37.881,-11.795]],"v":[[37.881,-16.664],[37.67,-22.816],[36.352,-26.068],[33.425,-28.31],[29.32,-29.057],[25.102,-28.266],[22.228,-25.998],[20.962,-22.737],[20.76,-16.664],[20.76,-11.795],[20.971,-5.643],[22.289,-2.391],[25.216,-0.149],[29.32,0.598],[33.539,-0.193],[36.413,-2.461],[37.679,-5.722],[37.881,-11.795]],"c":true},"ix":2},"nm":"O","mn":"ADBE Vector Shape - Group","hd":false},{"ind":1,"ty":"sh","ix":2,"ks":{"a":0,"k":{"i":[[30.48,-8.068],[30.410999999999998,-5.332],[29.807000000000002,-4.219],[28.497,-4.412000000000001],[28.16,-6.205],[28.16,-21.146],[28.263,-23.493000000000002],[28.968,-24.24],[30.114,-24.056],[30.48,-22.470000000000002]],"o":[[30.48,-6.3919999999999995],[30.129,-4.441999999999999],[28.811,-4.219],[28.226999999999997,-5.186],[28.16,-7.857],[28.16,-22.611],[28.673,-24.091],[29.812,-24.24],[30.407,-23.318],[30.48,-21.146]],"v":[[30.48,-8.068],[30.27,-4.887],[29.303,-4.219],[28.362,-4.799],[28.16,-7.857],[28.16,-21.146],[28.468,-23.792],[29.355,-24.24],[30.261,-23.687],[30.48,-21.146]],"c":true},"ix":2},"nm":"O","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"mm","mm":1,"nm":"Merge Paths 1","mn":"ADBE Vector Filter - Merge","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":2,"ix":5},"lc":1,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":8},"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":true},{"ty":"fl","c":{"a":0,"k":[0.92,0.92,0.92,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"O","np":5,"cix":2,"ix":2,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[46.986,0],[56.32,0],[59.414,-0.09699999999999998],[61.910000000000004,-0.8240000000000001],[63.525,-2.4699999999999998],[64.125,-5.9],[64.125,-18.422],[64.04299999999999,-22.924999999999997],[63.304,-25.6],[61.034,-27.639],[56.097,-28.459],[46.986,-28.459]],"o":[[46.986,0],[58.089,0],[61.172,-0.483],[63.106,-1.796],[64.005,-4.192],[64.125,-8.455],[64.125,-21.117],[63.714999999999996,-24.765],[61.992000000000004,-27.1],[58.514,-28.294999999999998],[52.523,-28.459],[46.986,-28.459]],"v":[[46.986,0],[56.32,0],[60.293,-0.29],[62.508,-1.31],[63.765,-3.331],[64.125,-8.455],[64.125,-18.422],[63.879,-23.845],[62.648,-26.35],[59.774,-27.967],[52.523,-28.459],[46.986,-28.459]],"c":true},"ix":2},"nm":"D","mn":"ADBE Vector Shape - Group","hd":false},{"ind":1,"ty":"sh","ix":2,"ks":{"a":0,"k":{"i":[[55.746,-23.505000000000003],[56.519999999999996,-22.898],[56.725,-21.346],[56.725,-9],[56.601,-5.939],[55.453,-4.869],[54.387,-23.59]],"o":[[56.332,-23.165],[56.684,-22.172],[56.725,-20.057],[56.725,-7.102],[56.108999999999995,-5.083],[54.387,-4.869],[55.196,-23.59]],"v":[[56.039,-23.335],[56.602,-22.535],[56.725,-20.057],[56.725,-9],[56.355,-5.511],[54.387,-4.869],[54.387,-23.59]],"c":true},"ix":2},"nm":"D","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"mm","mm":1,"nm":"Merge Paths 1","mn":"ADBE Vector Filter - Merge","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":2,"ix":5},"lc":1,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":8},"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":true},{"ty":"fl","c":{"a":0,"k":[0.92,0.92,0.92,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"D","np":5,"cix":2,"ix":3,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[68.981,-28.459],[65.206,0],[72.857,0],[73.302,-5.115],[75.949,-5.115],[76.346,0],[83.909,0],[79.677,-28.459]],"o":[[68.981,-28.459],[65.206,0],[72.857,0],[73.302,-5.115],[75.949,-5.115],[76.346,0],[83.909,0],[79.677,-28.459]],"v":[[68.981,-28.459],[65.206,0],[72.857,0],[73.302,-5.115],[75.949,-5.115],[76.346,0],[83.909,0],[79.677,-28.459]],"c":true},"ix":2},"nm":"A","mn":"ADBE Vector Shape - Group","hd":false},{"ind":1,"ty":"sh","ix":2,"ks":{"a":0,"k":{"i":[[73.208,-10.16],[73.874,-16.664],[75.38,-13.383]],"o":[[73.401,-12.679],[75.005,-17.366999999999997],[75.755,-10.16]],"v":[[73.208,-10.16],[74.628,-22.113],[75.755,-10.16]],"c":true},"ix":2},"nm":"A","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"mm","mm":1,"nm":"Merge Paths 1","mn":"ADBE Vector Filter - Merge","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":2,"ix":5},"lc":1,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":8},"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":true},{"ty":"fl","c":{"a":0,"k":[0.92,0.92,0.92,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"A","np":5,"cix":2,"ix":4,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[81.914,-28.459],[81.914,-22.764],[86.291,-22.764],[86.291,0],[93.691,0],[93.691,-22.764],[98.086,-22.764],[98.086,-28.459]],"o":[[81.914,-28.459],[81.914,-22.764],[86.291,-22.764],[86.291,0],[93.691,0],[93.691,-22.764],[98.086,-22.764],[98.086,-28.459]],"v":[[81.914,-28.459],[81.914,-22.764],[86.291,-22.764],[86.291,0],[93.691,0],[93.691,-22.764],[98.086,-22.764],[98.086,-28.459]],"c":true},"ix":2},"nm":"T","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":2,"ix":5},"lc":1,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":8},"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":true},{"ty":"fl","c":{"a":0,"k":[0.92,0.92,0.92,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"T","np":3,"cix":2,"ix":5,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[100.428,-28.459],[96.653,0],[104.304,0],[104.749,-5.115],[107.396,-5.115],[107.793,0],[115.356,0],[111.125,-28.459]],"o":[[100.428,-28.459],[96.653,0],[104.304,0],[104.749,-5.115],[107.396,-5.115],[107.793,0],[115.356,0],[111.125,-28.459]],"v":[[100.428,-28.459],[96.653,0],[104.304,0],[104.749,-5.115],[107.396,-5.115],[107.793,0],[115.356,0],[111.125,-28.459]],"c":true},"ix":2},"nm":"A","mn":"ADBE Vector Shape - Group","hd":false},{"ind":1,"ty":"sh","ix":2,"ks":{"a":0,"k":{"i":[[104.655,-10.16],[105.321,-16.664],[106.827,-13.383]],"o":[[104.848,-12.679],[106.452,-17.366999999999997],[107.202,-10.16]],"v":[[104.655,-10.16],[106.075,-22.113],[107.202,-10.16]],"c":true},"ix":2},"nm":"A","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"mm","mm":1,"nm":"Merge Paths 1","mn":"ADBE Vector Filter - Merge","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":2,"ix":5},"lc":1,"lj":1,"ml":4,"ml2":{"a":0,"k":4,"ix":8},"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":true},{"ty":"fl","c":{"a":0,"k":[0.92,0.92,0.92,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"A","np":5,"cix":2,"ix":6,"mn":"ADBE Vector Group","hd":false}],"ip":35.0000014255792,"op":106.000004317469,"st":25.0000010182709,"bm":0,"completed":true}],"markers":[],"__complete":true}

metadata.json

@@ -0,0 +1,267 @@
+{
+    "tables": {
+        "brands": {
+            "columns": {
+                "brand_id": {
+                    "sdtype": "id"
+                },
+                "brand_name": {
+                    "sdtype": "unknown",
+                    "pii": true
+                }
+            },
+            "primary_key": "brand_id"
+        },
+        "categories": {
+            "columns": {
+                "category_id": {
+                    "sdtype": "id"
+                },
+                "category_name": {
+                    "sdtype": "unknown",
+                    "pii": true
+                }
+            },
+            "primary_key": "category_id"
+        },
+        "customers": {
+            "columns": {
+                "customer_id": {
+                    "sdtype": "id"
+                },
+                "first_name": {
+                    "sdtype": "first_name",
+                    "pii": true
+                },
+                "last_name": {
+                    "sdtype": "last_name",
+                    "pii": true
+                },
+                "phone": {
+                    "sdtype": "categorical"
+                },
+                "email": {
+                    "sdtype": "email"
+                },
+                "street": {
+                    "sdtype": "unknown",
+                    "pii": true
+                },
+                "city": {
+                    "sdtype": "city",
+                    "pii": true
+                },
+                "state": {
+                    "sdtype": "administrative_unit",
+                    "pii": true
+                },
+                "zip_code": {
+                    "sdtype": "postcode",
+                    "pii": true
+                }
+            },
+            "primary_key": "customer_id"
+        },
+        "orders": {
+            "columns": {
+                "order_id": {
+                    "sdtype": "id"
+                },
+                "customer_id": {
+                    "sdtype": "id"
+                },
+                "order_status": {
+                    "sdtype": "categorical"
+                },
+                "order_date": {
+                    "sdtype": "datetime",
+                    "datetime_format": "%Y-%m-%d"
+                },
+                "required_date": {
+                    "sdtype": "datetime",
+                    "datetime_format": "%Y-%m-%d"
+                },
+                "shipped_date": {
+                    "sdtype": "datetime",
+                    "datetime_format": "%Y-%m-%d"
+                },
+                "store_id": {
+                    "sdtype": "categorical"
+                },
+                "staff_id": {
+                    "sdtype": "id"
+                }
+            },
+            "primary_key": "order_id"
+        },
+        "order_items": {
+            "columns": {
+                "order_id": {
+                    "sdtype": "id"
+                },
+                "item_id": {
+                    "sdtype": "categorical"
+                },
+                "product_id": {
+                    "sdtype": "id"
+                },
+                "quantity": {
+                    "sdtype": "categorical"
+                },
+                "list_price": {
+                    "sdtype": "numerical"
+                },
+                "discount": {
+                    "sdtype": "numerical"
+                }
+            }
+        },
+        "products": {
+            "columns": {
+                "product_id": {
+                    "sdtype": "id"
+                },
+                "product_name": {
+                    "sdtype": "unknown",
+                    "pii": true
+                },
+                "brand_id": {
+                    "sdtype": "id"
+                },
+                "category_id": {
+                    "sdtype": "id"
+                },
+                "model_year": {
+                    "sdtype": "categorical"
+                },
+                "list_price": {
+                    "sdtype": "numerical"
+                }
+            },
+            "primary_key": "product_id"
+        },
+        "staffs": {
+            "columns": {
+                "staff_id": {
+                    "sdtype": "id"
+                },
+                "first_name": {
+                    "sdtype": "first_name",
+                    "pii": true
+                },
+                "last_name": {
+                    "sdtype": "last_name",
+                    "pii": true
+                },
+                "email": {
+                    "sdtype": "email"
+                },
+                "phone": {
+                    "sdtype": "unknown",
+                    "pii": true
+                },
+                "active": {
+                    "sdtype": "categorical"
+                },
+                "store_id": {
+                    "sdtype": "numerical"
+                },
+                "manager_id": {
+                    "sdtype": "numerical"
+                }
+            },
+            "primary_key": "staff_id"
+        },
+        "stocks": {
+            "columns": {
+                "store_id": {
+                    "sdtype": "categorical"
+                },
+                "product_id": {
+                    "sdtype": "id"
+                },
+                "quantity": {
+                    "sdtype": "numerical"
+                }
+            }
+        },
+        "stores": {
+            "columns": {
+                "store_id": {
+                    "sdtype": "numerical"
+                },
+                "store_name": {
+                    "sdtype": "categorical"
+                },
+                "phone": {
+                    "sdtype": "categorical"
+                },
+                "email": {
+                    "sdtype": "email",
+                    "pii": true
+                },
+                "street": {
+                    "sdtype": "categorical"
+                },
+                "city": {
+                    "sdtype": "city",
+                    "pii": true
+                },
+                "state": {
+                    "sdtype": "administrative_unit",
+                    "pii": true
+                },
+                "zip_code": {
+                    "sdtype": "postcode",
+                    "pii": true
+                }
+            },
+            "primary_key": "email"
+        }
+    },
+    "relationships": [
+        {
+            "parent_table_name": "brands",
+            "child_table_name": "products",
+            "parent_primary_key": "brand_id",
+            "child_foreign_key": "brand_id"
+        },
+        {
+            "parent_table_name": "categories",
+            "child_table_name": "products",
+            "parent_primary_key": "category_id",
+            "child_foreign_key": "category_id"
+        },
+        {
+            "parent_table_name": "customers",
+            "child_table_name": "orders",
+            "parent_primary_key": "customer_id",
+            "child_foreign_key": "customer_id"
+        },
+        {
+            "parent_table_name": "orders",
+            "child_table_name": "order_items",
+            "parent_primary_key": "order_id",
+            "child_foreign_key": "order_id"
+        },
+        {
+            "parent_table_name": "products",
+            "child_table_name": "order_items",
+            "parent_primary_key": "product_id",
+            "child_foreign_key": "product_id"
+        },
+        {
+            "parent_table_name": "products",
+            "child_table_name": "stocks",
+            "parent_primary_key": "product_id",
+            "child_foreign_key": "product_id"
+        },
+        {
+            "parent_table_name": "staffs",
+            "child_table_name": "orders",
+            "parent_primary_key": "staff_id",
+            "child_foreign_key": "staff_id"
+        }
+    ],
+    "METADATA_SPEC_VERSION": "MULTI_TABLE_V1"
+}

tbl_dtl.csv

@@ -0,0 +1,5 @@
+Table Name,ETL Timestamp,Change Timestamp,ETL Filter
+TPM,TPM._hoodie_commit_time,TPM.prev_updt_ts,"coalesce(OPERATION,'NA')<>'D'"
+TPP,TPP._hoodie_commit_time,TPP.prev_updt_ts,"coalesce(OPERATION,'NA')<>'D'"
+TSM,TSM._hoodie_commit_time,TSM.prev_updt_ts,"coalesce(OPERATION,'NA')<>'D'"
+TRM,TRM._hoodie_commit_time,TRM.prev_updt_ts,"coalesce(OPERATION,'NA')<>'D'"

template.txt

@@ -0,0 +1,254 @@
+import traceback
+from pyspark.sql import SparkSession
+from pyspark import SparkConf
+from pyspark.sql.functions import col,regexp_replace, concat_ws, when, collect_list, lit, to_timestamp
+from pyspark.sql.functions import year, month, date_format
+from pyspark.sql import functions as F
+from pyspark.sql.types import LongType,DecimalType,IntegerType,TimestampType,DoubleType
+from pyspark.sql.functions import *
+from pytz import timezone
+from datetime import datetime,timedelta
+from pyspark.sql.window import Window
+import json
+import sys
+import logging
+import datetime
+import time
+import os
+import psycopg2
+import requests
+from requests.auth import HTTPBasicAuth
+import base64
+import functools
+import boto3
+
+# adding '/home/hadoop' path of emr master instance as our downloaded packages will be present at this path
+sys.path.append('/home/hadoop')
+
+curr_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+log_file_name = 'job_' + str(datetime.datetime.now().strftime('%Y%m%d_%H%M%S_%f')) + '.log'
+extra = {'log_file_name': log_file_name}
+logger = logging.getLogger(__name__)
+syslog = logging.FileHandler(log_file_name, mode='w')
+formatter = logging.Formatter('%(log_file_name)s;%(asctime)s;%(levelname)s;%(message)s')
+syslog.setFormatter(formatter)
+logger.setLevel(logging.INFO)
+logger.addHandler(syslog)
+logger = logging.LoggerAdapter(logger, extra)
+
+def read_config(config_path):
+    logger.info("Inside read config")
+    try:
+        # checking if config path provided as input is s3 path or file system path
+        if config_path[0:2] == 's3':
+            # read config file from s3
+            logger.info("Reading config file from S3")
+            s3 = boto3.resource('s3')
+            file_object = s3.Object(config_path.split('/')[2], '/'.join(config_path.split('/')[3:]))
+            file_content = file_object.get()['Body'].read().decode('utf-8')
+            # converting file content to json format
+            json_content = json.loads(file_content)
+            json_object = json.dumps(json_content)
+        else:
+            # reading config file from system
+            logger.info("Reading config file from path : " + config_path)
+            # converting file content to json format
+            json_content = json.load(open(config_path, 'r'))
+            json_object = json.dumps(json_content)
+            logger.info("Input Config Details:")
+        logger.info(json_object)
+        return json_content
+    except Exception as e:
+        raise Exception("Error reading config.")
+
+def get_secret(secret):
+    secret_name = secret
+    region_name = "ap-south-1"
+
+    session = boto3.session.Session()
+    client = session.client(
+    service_name='secretsmanager',
+    region_name=region_name,
+    )
+
+    try:
+        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ResourceNotFoundException':
+            print("The requested secret " + secret_name + " was not found")
+        elif e.response['Error']['Code'] == 'InvalidRequestException':
+            print("The request was invalid due to:", e)
+        elif e.response['Error']['Code'] == 'InvalidParameterException':
+            print("The request had invalid params:", e)
+        elif e.response['Error']['Code'] == 'DecryptionFailure':
+            print("The requested secret can't be decrypted using the provided KMS key:", e)
+        elif e.response['Error']['Code'] == 'InternalServiceError':
+            print("An error occurred on service side:", e)
+    else:
+	# Secrets Manager decrypts the secret value using the associated KMS CMK
+	# Depending on whether the secret was a string or binary, only one of these fields will be populated
+        if 'SecretString' in get_secret_value_response:
+            text_secret_data = get_secret_value_response['SecretString']
+            return text_secret_data
+        else:
+            binary_secret_data = get_secret_value_response['SecretBinary']
+            return binary_secret_data
+    logger.info("Secret manager read complete")
+			
+def create_spark_session(config):
+    logger.info("Inside create spark session")
+    try:
+        conf = SparkConf()
+
+        # setting spark configuration properties provided in config file
+        spark_conf = dict(config['spark_properties'])
+        for key in spark_conf.keys():
+            conf.set(key, spark_conf[key])
+            logger.info("Secret manager read")
+        if 'application_name' in list(config.keys()):
+            if config['application_name'] != '':
+                app_name = config['application_name']
+            else:
+                app_name = 'DefaultApp'
+        else:
+            app_name = 'DefaultApp'
+        logger.info("Secret manager read start")
+        # creating spark session
+        spark = SparkSession.builder.config(conf=conf).appName(app_name).enableHiveSupport().getOrCreate()
+        spark.sparkContext.setLogLevel("ERROR")
+        spark.conf.set("spark.sql.autoBroadcastJoinThreshold",-1)
+        spark.conf.set("spark.sql.legacy.parquet.datetimeRebaseModeInRead",'LEGACY')
+        spark.conf.set("spark.sql.legacy.timeParserPolicy",'CORRECTED')
+        spark.conf.set("spark.sql.legacy.parquet.int96RebaseModeInWrite",'CORRECTED')
+        spark.conf.set("spark.sql.legacy.parquet.datetimeRebaseModeInWrite",'CORRECTED')
+        spark.conf.set("spark.sql.legacy.parquet.int96RebaseModeInRead",'CORRECTED')
+        spark.conf.set("spark.sql.shuffle.partitions",100)
+        logger.info("Spark session object created")
+        return spark
+    except Exception as e:
+        raise Exception("Error in Spark Session Creation.")
+        
+def read_file(spark,config,table):
+    readOptions = {
+      'hoodie.datasource.query.type': 'incremental',
+      'hoodie.datasource.hive_sync.support_timestamp': 'true'
+    }
+    path = config['Paths'][table]
+    df=spark.read.format("hudi").load(path)
+    df =df.withColumn('_hoodie_commit_time',to_timestamp(F.concat(F.substring(col('_hoodie_commit_time'),1,4),F.lit('-'),\
+                                                                                   F.substring(col('_hoodie_commit_time'),5,2),F.lit('-'),\
+                                                                                   F.substring(col('_hoodie_commit_time'),7,2),F.lit(' '),\
+                                                                                   F.substring(col('_hoodie_commit_time'),9,2),F.lit(':'),\
+                                                                                   F.substring(col('_hoodie_commit_time'),11,2),F.lit(':'),\
+                                                                                   F.substring(col('_hoodie_commit_time'),13,2)\
+                                                                                  )))
+    return df
+
+def get_max_audit_batch(conn,job_name, config):
+    cur = conn.cursor()
+    cur.execute("SELECT COALESCE(MAX(COALESCE(BATCH_ID,0)),0)+1 FROM "+config['audit_table'])
+    result = cur.fetchall()[0][0]
+    logger.info("Maximum batch id in Audit Table is :"+str(result))
+    return result
+
+def read_max_update_date(conn, job_name, table, config):
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT MAX(max_update_date) from "+config['audit_table']+" WHERE mart_table_name = '"+job_name+"' AND src_table_name = '"+table+"'")
+        query_results = cur.fetchall()
+    except Exception as e:
+        print("Database connection failed due to {}".format(e))
+        raise Exception("Error reading audit table.")
+    return query_results
+    logger.info("Reading max of max_update_date from audit table complete")
+
+def insert_max_update_date(spark,conn, job_name, table, max_update_date,source_reference_date, max_batch_id, config):
+    try:
+        cur = conn.cursor()
+        cur.execute("INSERT INTO "+config['audit_table']+"(mart_table_name, src_table_name, max_update_date, load_timestamp,source_reference_date,batch_id) VALUES ('"+str(job_name)+"', '"+str(table)+"', '"+str(max_update_date)+"', SYSDATE ,'"+str(source_reference_date)+"' as source_reference_date,cast('"+str(max_batch_id)+"' as int) as batch_id)")
+
+    except Exception as e:
+        print("Database connection failed due to {}".format(e))
+        raise Exception("Error Updating audit table.")
+    logger.info("Inserting max max_update_date  into audit table complete")
+	
+def write_file(spark,conn,redshift_iam_role,resultdf_path, config, table_name):
+    #Writing resultant data into incr table using copy command
+    logger.info("write data to redshift started")
+    try:
+        cur = conn.cursor()
+        cur.execute(f"""Truncate table int.{table_name};commit;""" )
+        sql="""COPY %s FROM '%s' credentials 'aws_iam_role=%s' FORMAT PARQUET; commit;""" % \
+             (f"int.{table_name}", resultdf_path,redshift_iam_role)
+        cur.execute(sql)
+ 
+    except Exception as e:
+        print("Database connection failed due to {}".format(e))
+        raise Exception("Error Inserting target table.")
+    print("write complete")
+    logger.info("upsert data to rds completed")
+
+def main():
+    logger.info("Inside main function")
+    if len(sys.argv) != 2:
+        logger.info(len(sys.argv))
+        logger.info("Command line arguments : " + str(sys.argv))
+        logger.info("Incorrect command line arguments.")
+        exit(1)
+
+    config = {}
+    spark = ''
+    job_status = ''
+
+    try:
+        # reading json config file
+        logger.info("Calling function to read config file")
+        config = read_config(sys.argv[1])
+        logger.info("Calling function to create Spark session object")
+        #creating spark session
+        spark = create_spark_session(config)
+        logger.info("Calling function to read input file")
+        start_time = datetime.datetime.now(timezone("Asia/Kolkata")).strftime('%Y-%m-%d %H:%M:%S')
+        
+        #creating redshift database connection
+        redshift_secret = get_secret(config['redshift_secret'])
+        redshift_secret = json.loads(redshift_secret)
+        redshift_user = redshift_secret['username']
+        redshift_pwd = redshift_secret['password']
+        redshift_host = redshift_secret['host']
+        redshift_port = str(redshift_secret['port'])
+        redshift_dbname = redshift_secret['dbname']
+        #creating database connection
+        redshift_conn=psycopg2.connect(dbname=redshift_dbname, host=redshift_host, port=redshift_port, user=redshift_user, password=redshift_pwd)
+        redshift_dburl = "jdbc:postgresql://"+redshift_host+":"+redshift_port+"/"+redshift_dbname
+        cur = redshift_conn.cursor()
+        max_batch_id = get_max_audit_batch(redshift_conn, config['application_name'], config)
+
+        INSERT_CODE_1        
+        
+        #writing from parquet to table in database
+        write_file(spark, redshift_conn, config['redshift_iam_role'],config['incr2df_path'],config, config['incr2df']) 
+        write_file(spark, redshift_conn, config['redshift_iam_role'],config['resultdf_path'],config, config['resultdf'])
+        
+        INSERT_CODE_2
+        
+        print('Run Successful')
+        print('End of Code')
+
+    
+    except Exception as e:
+        #job gets error
+        job_status = 'Failed'
+        print(e)
+
+    finally:
+        spark.catalog.clearCache()
+        redshift_conn.commit()
+        redshift_conn.close()
+        spark.stop()
+ 
+
+if __name__ == "__main__":
+    # calling main function
+    logger.info("Calling main function")
+    main()

ydata_config.yml

@@ -0,0 +1,219 @@
+# Title of the document
+title: "Pandas Profiling Report"
+
+# Metadata
+dataset:
+  description: ""
+  creator: ""
+  author: ""
+  copyright_holder: ""
+  copyright_year: ""
+  url: ""
+
+variables:
+  descriptions: {}
+
+# infer dtypes
+infer_dtypes: false
+
+# Show the description at each variable (in addition to the overview tab)
+show_variable_description: true
+
+# Number of workers (0=multiprocessing.cpu_count())
+pool_size: 0
+
+# Show the progress bar
+progress_bar: true
+
+# Per variable type description settings
+vars:
+    num:
+        quantiles:
+              - 0.05
+              - 0.25
+              - 0.5
+              - 0.75
+              - 0.95
+        skewness_threshold: 20
+        low_categorical_threshold: 5
+        # Set to zero to disable
+        chi_squared_threshold: 0.0
+    cat:
+        length: false
+        characters: false
+        words: false
+        cardinality_threshold: 50
+        n_obs: 5
+        # Set to zero to disable
+        chi_squared_threshold: 0.0
+        coerce_str_to_date: false
+        redact: false
+        histogram_largest: 10
+        stop_words: []
+
+    bool:
+        n_obs: 3
+        # string to boolean mapping dict
+        mappings:
+            t: true
+            f: false
+            yes: true
+            no: false
+            y: true
+            n: false
+            true: true
+            false: false
+    path:
+        active: false
+    file:
+        active: false
+    image:
+        active: false
+        exif: false
+        hash: false
+    url:
+        active: false
+    timeseries:
+        active: false
+        autocorrelation: 0.7
+        lags: [1, 7, 12, 24, 30]
+        significance: 0.05
+        pacf_acf_lag: 100
+
+# Sort the variables. Possible values: "ascending", "descending" or null (leaves original sorting)
+sort: null
+
+# which diagrams to show
+missing_diagrams:
+    bar: false
+    matrix: false
+    heatmap: false
+
+correlations:
+    pearson:
+      calculate: false
+      warn_high_correlations: true
+      threshold: 0.9
+    spearman:
+      calculate: false
+      warn_high_correlations: false
+      threshold: 0.9
+    kendall:
+      calculate: false
+      warn_high_correlations: false
+      threshold: 0.9
+    phi_k:
+      calculate: false
+      warn_high_correlations: false
+      threshold: 0.9
+    cramers:
+      calculate: false
+      warn_high_correlations: true
+      threshold: 0.9
+    auto:
+       calculate: false
+       warn_high_correlations: true
+       threshold: 0.9
+
+
+# Bivariate / Pairwise relations
+interactions:
+  targets: []
+  continuous: false
+
+# For categorical
+categorical_maximum_correlation_distinct: 100
+
+report:
+  precision: 10
+
+# Plot-specific settings
+plot:
+    # Image format (svg or png)
+    image_format: "svg"
+    dpi: 800
+
+    scatter_threshold: 1000
+
+    correlation:
+        cmap: 'RdBu'
+        bad: '#000000'
+
+    missing:
+        cmap: 'RdBu'
+        # Force labels when there are > 50 variables
+        force_labels: true
+
+    cat_frequency:
+        show: true  # if false, the category frequency plot is turned off
+        type: 'bar' # options: 'bar', 'pie'
+        max_unique: 0
+        colors: null # use null for default or give a list of matplotlib recognised strings
+
+    histogram:
+        x_axis_labels: true
+
+        # Number of bins (set to 0 to automatically detect the bin size)
+        bins: 50
+
+        # Maximum number of bins (when bins=0)
+        max_bins: 250
+
+    font_path: null
+
+# The number of observations to show
+n_obs_unique: 5
+n_extreme_obs: 5
+n_freq_table_max: 10
+
+# Use `deep` flag for memory_usage
+memory_deep: false
+
+# Configuration related to the duplicates
+duplicates:
+    head: 0
+    key: "# duplicates"
+
+# Configuration related to the samples area
+samples:
+    head: 0
+    tail: 0
+    random: 0
+
+# Configuration related to the rejection of variables
+reject_variables: true
+
+# When in a Jupyter notebook
+notebook:
+    iframe:
+        height: '800px'
+        width: '100%'
+        # or 'src'
+        attribute: 'srcdoc'
+
+html:
+    # Minify the html
+    minify_html: true
+
+    # Offline support
+    use_local_assets: true
+
+    # If true, single file, else directory with assets
+    inline: true
+
+    # Show navbar
+    navbar_show: true
+
+    # Assets prefix if inline = true
+    assets_prefix: null
+
+    # Styling options for the HTML report
+    style:
+      theme: null
+      logo: ""
+      primary_colors:
+      - "#377eb8"
+      - "#e41a1c"
+      - "#4daf4a"
+
+    full_width: false