satendrakumar’s gists

satendrakumar / s3_utils.py

Last active November 8, 2018 12:58

Upload and download from/to S3 using boto3 Api

	import boto3

	s3 = boto3.resource('s3')


	def download(bucket_name, key, filePath):
	try:
	s3.meta.client.download_file(bucket_name, key, filePath)
	except ValueError as err:
	print('Error:', err)

satendrakumar / Loops.scala

Last active August 26, 2017 06:49

while loop alternatives in Scala

	import java.io._

	object Loop1 extends App {
	//Java way
	val inputStream: BufferedInputStream = new BufferedInputStream(new FileInputStream("input.csv"))
	val outputStream = new BufferedOutputStream(new FileOutputStream("output.csv"))
	val buffer = new Array[Byte](32 * 1024)
	var bytesRead: Int = inputStream.read(buffer)
	while (bytesRead > 0) {
	println("writing.......")

satendrakumar / FoldLeft.scala

Created June 11, 2017 12:17

Capabilities of foldLeft of Scala



	object FoldLeft extends App {

	def reverse(list: List[Int]): List[Int] =
	list.foldLeft[List[Int]](Nil)((acc, element) => element :: acc)

	def dedupe(list: List[Int]): List[Int] = {
	list.foldLeft[List[Int]](Nil)((acc, element) => if (acc.contains(element)) acc else acc :+ element)
	}

satendrakumar / RemoveHeaders.scala

Last active May 10, 2017 07:49

Remove headers from csv files without comparing each record using Spark

	import java.io.{BufferedReader, File, InputStreamReader}
	import javax.annotation.concurrent.NotThreadSafe

	import org.apache.hadoop.io.{LongWritable, Text}
	import org.apache.hadoop.mapred.{FileSplit, TextInputFormat}
	import org.apache.spark.broadcast.Broadcast
	import org.apache.spark.rdd.HadoopRDD
	import org.apache.spark.{SparkConf, SparkContext}

	@NotThreadSafe

satendrakumar / CSVParser.scala

Last active May 13, 2017 11:12

Parse CSV using univocity parser

	import com.univocity.parsers.csv.{CsvParser, CsvParserSettings}


	class CSVParser(delimiter: Char = ',') {

	private val parser = {
	val settings = new CsvParserSettings
	val format = settings.getFormat
	format.setLineSeparator("\n")
	format.setDelimiter(delimiter)

satendrakumar / Logging.scala

Last active April 15, 2017 04:36

Cleaner way to use Logback logger

	import org.slf4j.{Logger, LoggerFactory}

	trait Logging {

	protected val logger: Logger = LoggerFactory.getLogger(this.getClass())

	protected def debug(message: String): Unit = logger.debug(message)

	protected def debug(message: String, exception: Throwable): Unit = logger.debug(message, exception)

satendrakumar / JsonUtility.scala

Last active February 12, 2019 06:32

Scala Utility class for parsing and writing Json(Write once and use everywhere )


	import org.json4s._
	import org.json4s.native.{JsonMethods, Serialization}

	object JsonUtility {

	implicit val formats = DefaultFormats

	def write[T <: AnyRef](value: T): String = Serialization.write(value)

satendrakumar / haskell-forall-implementation

Last active March 26, 2017 08:20

	Prelude> let forall :: (a -> Bool) -> [a] -> Bool ; forall f []= True; forall f (x:xs) = f x && forall f xs
	Prelude> forall even [2,4,6]
	True
	Prelude> forall even [2,4,5]
	False
	Prelude> let forall :: (a -> Bool) -> [a] -> Bool ; forall f []= True; forall f (x:xs) = if (f x) then forall f xs else False
	Prelude> forall even [2,4,5]
	False
	Prelude> forall even [2,4,6]
	True

satendrakumar / GenericTypeConverter.scala

Created March 23, 2017 12:44

Generic Type Converter

	object GenericTypeConverter extends App{

	def convert[T: ClassTag](value:Any): Option[T] = {
	val ct = implicitly[ClassTag[T]]
	value match {
	case ct(x) => Some(x)
	case _ => None
	}
	}

satendrakumar / JavaNullWithScala.scala

Last active April 15, 2017 04:32

Handling Java null with Scala

	object play {
	println("Welcome to the Scala worksheet")

	case class User(id:Int, name:String)
	def getUser:User = null
	def processUser(u:User):String= u.name
	Option(getUser)

	Option(getUser).fold("Sky")(u=>processUser(u))

Satendra Kumar satendrakumar