kshitijcode
kshitijcode
/ gist:af9f179db7298bcad1572a1ce068aea9
Created
March 19, 2024 10:04
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import aiohttp | |
| import argparse | |
| import asyncio | |
| import json | |
| import logging | |
| import re | |
| import tiktoken | |
| import time | |
| from status_tracker import StatusTracker | |
| from api_request import APIRequest |
kshitijcode
/ service.yaml
Created
January 9, 2023 09:39
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| annotations: | |
| service.beta.kubernetes.io/azure-load-balancer-internal: "true" | |
| #https://cloud-provider-azure.sigs.k8s.io/topics/pls-integration/ | |
| service.beta.kubernetes.io/azure-pls-create: "true" | |
| #create private link service with name private-link-demo | |
| service.beta.kubernetes.io/azure-pls-name: "private-link-demo" |
kshitijcode
/ successandassert
Created
November 30, 2021 04:32
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| pipeline_run_result = pipelineutils.run_and_observe_pipeline( | |
| azure_credential, synapse_endpoint, pipeline_name, masterpipeline_processed_params) | |
| assert pipeline_run_result == constants.PIPELINE_SUCCESS_STATUS | |
| # Check for Data in processed Zone | |
| cursor = sql_connection_client.cursor() | |
| cursor.execute( | |
| "SELECT COUNT(*) AS COUNT FROM [dbo].[{0}]".format(table_processed)) | |
| row = cursor.fetchone() |
kshitijcode
/ rawtoprocessed
Created
November 30, 2021 04:31
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| table_raw = "integrationtest_users" | |
| table_processed = "processed_integrationtest_user" | |
| # Trigger the Master Pipeline for Processed to processed Zone | |
| masterpipeline_processed_params = { | |
| "lookUpTables": [{ | |
| "SourceTableSchemaName": "dbo", | |
| "SourceTableName": table_raw, | |
| "SinkTableSchemaName": "dbo", | |
| "SinkTableName": table_processed, |
kshitijcode
/ rawtoprocessedparams
Created
November 30, 2021 04:30
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| "lookUpTables": [{ | |
| "SourceTableSchemaName": "dbo", | |
| "SourceTableName": table_raw, | |
| "SinkTableSchemaName": "dbo", | |
| "SinkTableName": table_processed, | |
| "HasIncrementalData": "false" | |
| }], | |
| "sourceDatabase": database, |
kshitijcode
/ rawzoneassertions
Created
November 30, 2021 04:30
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| assert pipeline_run_result == constants.PIPELINE_SUCCESS_STATUS | |
| # Check for Data in Raw Zone | |
| parquet_dataframe = adls.read_parquet_file_from_ADLS( | |
| adls_connection_client, container_name, target_path) | |
| num_of_rows = len(parquet_dataframe.index) | |
| # Assert | |
| print(f"Number of Rows Fetched : { num_of_rows }\n") | |
| assert num_of_rows >=1 |
kshitijcode
/ pipelineinvoke
Created
November 30, 2021 04:29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| pipeline_run_result = pipelineutils.run_and_observe_pipeline( | |
| azure_credential, synapse_endpoint, pipeline_name, masterpipeline_raw_params) |
kshitijcode
/ parameterssoucetoraw
Created
November 30, 2021 04:28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| "basePath": base_path, | |
| "filePath": "user_*.parquet", | |
| "targetTable": target_table, | |
| "badDataTable": "bad_users", | |
| "containerName": container_name, | |
| "archivePath": "archive", | |
| "storageAccountName": storage_account_name, | |
| "readFromSparkTables": False, | |
| "database": database, |
kshitijcode
/ user.parquet
Created
November 30, 2021 04:25
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"registration_dttm":28529000000000,"id":1,"first_name":"Amanda","last_name":"Jordan","email":"[email protected]","gender":"Female","ip_address":"1.197.201.2","cc":"6759521864920116","country":"Indonesia","birthdate":"3/8/1971","salary":49756.53,"title":"Internal Auditor","comments":"1E+02"} | |
| {"registration_dttm":61443000000000,"id":2,"first_name":"Albert","last_name":"Freeman","email":"[email protected]","gender":"Male","ip_address":"218.111.175.34","cc":"","country":"Canada","birthdate":"1/16/1968","salary":150280.17,"title":"Accountant IV","comments":""} | |
| {"registration_dttm":4171000000000,"id":3,"first_name":"Evelyn","last_name":"Morgan","email":"[email protected]","gender":"Female","ip_address":"7.161.136.94","cc":"6767119071901597","country":"Russia","birthdate":"2/1/1960","salary":144972.51,"title":"Structural Engineer","comments":""} | |
| {"registration_dttm":2181000000000,"id":4,"first_name":"Denise","last_name":"Riley","email":"[email protected]","gender":"Female","ip_address":"140.35.109.83","cc":"35760 |
kshitijcode
/ user.parquet.as.json
Created
November 30, 2021 04:25
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"registration_dttm":28529000000000,"id":1,"first_name":"Amanda","last_name":"Jordan","email":"[email protected]","gender":"Female","ip_address":"1.197.201.2","cc":"6759521864920116","country":"Indonesia","birthdate":"3/8/1971","salary":49756.53,"title":"Internal Auditor","comments":"1E+02"} | |
| {"registration_dttm":61443000000000,"id":2,"first_name":"Albert","last_name":"Freeman","email":"[email protected]","gender":"Male","ip_address":"218.111.175.34","cc":"","country":"Canada","birthdate":"1/16/1968","salary":150280.17,"title":"Accountant IV","comments":""} | |
| {"registration_dttm":4171000000000,"id":3,"first_name":"Evelyn","last_name":"Morgan","email":"[email protected]","gender":"Female","ip_address":"7.161.136.94","cc":"6767119071901597","country":"Russia","birthdate":"2/1/1960","salary":144972.51,"title":"Structural Engineer","comments":""} | |
| {"registration_dttm":2181000000000,"id":4,"first_name":"Denise","last_name":"Riley","email":"[email protected]","gender":"Female","ip_address":"140.35.109.83","cc":"35760 |
NewerOlder