Shritam Kumar Mund ishritam

🎯

Focusing

Data Engineer, Python Programmer, Machine Learner, Deep Learner

ishritam / zalando_etl.py

Last active June 20, 2021 12:10

Airflow DAG file For Zalando

	import json
	import requests
	import pandas as pd
	from airflow.models import DAG
	from airflow.operators.python import PythonOperator
	from datetime import datetime


	default_args = {
	'start_date': datetime(year=2021, month=6, day=20)

ishritam / load.py

Created June 20, 2021 11:25

Zalando ETL Load

	def load_data(path: str, ti) -> None:
	data = ti.xcom_pull(key='transformed_data', task_ids=['transform_data'])
	data_df = pd.DataFrame(data[0])
	data_df.to_csv(path, index=None)

ishritam / transform.py

Created June 20, 2021 11:22

ETL_zalando_transform

	def transform_data(ti) -> None:
	data = ti.xcom_pull(key='extracted_data', task_ids=['extract_data'])[0]
	transformed_data = []
	for item in data:
	transformed_data.append({
	'sku': item.get("sku", ""),
	'Name': item['name'],
	'Price': item['price'].get("original"),
	'Brand Name': item['brand_name'],
	'Thumbnail': f"https://img01.ztat.net/article/{item['media'][0]['path']}",

ishritam / extraction.py

Created June 20, 2021 11:10

Airflow Zalando Extraction

	def extract_data(url: str, headers: str, ti) -> None:
	res = requests.get(url, headers=headers)
	json_data = json.loads(res.content)['articles']
	ti.xcom_push(key='extracted_data', value=json_data)

ishritam / Term_Extraction.py

Created October 13, 2020 17:27

	# coding=UTF-8
	import pickle
	import nltk
	from nltk.corpus import brown
	#TextBlob FastNPExtractor + ConllExtractor
	# Textblob
	from textblob import TextBlob
	from textblob.np_extractors import FastNPExtractor
	from textblob.np_extractors import ConllExtractor

ishritam / wildcard_api.py

Created August 17, 2020 13:17

	try:
	from flask import app,Flask
	from flask_restful import Resource, Api, reqparse
	import elasticsearch
	from elasticsearch import Elasticsearch
	import datetime
	import concurrent.futures
	import requests
	import json
	except Exception as e:

ishritam / elk_upload.py

Last active August 17, 2020 12:54

	from elasticsearch import helpers
	res = helpers.bulk(es,gen(df),request_timeout= 300)

ishritam / shape_of_data_es.py

Created August 17, 2020 12:46

ishritam / mapping.py

Created August 17, 2020 12:40

ishritam / csv_to_dict.py

Created August 17, 2020 12:36