prongs’s gists

prongs / start-datanode.sh

Last active April 8, 2018 09:07

start-hadoop

	apt-get -y update
	apt-get update

	apt-get -y install --no-install-recommends \
	apt-transport-https \
	curl \
	software-properties-common

	apt-get -y install -y --no-install-recommends \
	linux-image-extra-$(uname -r) \

prongs / setup_lens.bash

Created June 21, 2016 13:10

	git clone https://github.com/inmobi/hive
	cd hive
	git fetch origin
	git checkout hive-release-2.1.1-inm
	mvn clean install -DskipTests -Pdist,deb
	cd packaging/target/bin/bin
	HIVE_HOME=`pwd`
	cd -
	echo export HIVE_HOME=$HIVE_HOME > ~/.bashrc # or whatever file
	cd ..

prongs / init.java

Last active January 18, 2016 12:17

	public static final String XML_LICENSE_HEADER = "<!--\n" +
	"\n" +
	" Licensed to the Apache Software Foundation (ASF) under one\n" +
	" or more contributor license agreements. See the NOTICE file\n" +
	" distributed with this work for additional information\n" +
	" regarding copyright ownership. The ASF licenses this file\n" +
	" to you under the Apache License, Version 2.0 (the\n" +
	" \"License\"); you may not use this file except in compliance\n" +
	" with the License. You may obtain a copy of the License at\n" +
	"\n" +

prongs / error

Created July 21, 2015 13:28

	2015-07-21 18:53:39,998 WARN [main]: common.LogUtils (LogUtils.java:logConfigLocation(145)) - hive-site.xml not found on CLASSPATH
	2015-07-21 18:53:40,011 INFO [main]: mr.ExecDriver (SessionState.java:printInfo(563)) - Execution log at: /var/folders/m6/05_cdz4d6dn1jh51c_dyg46mhr0n9j/T//rajat.khandelwal/rajat.khandelwal_20150721185353_1289f2f0-6bd8-4df7-9a9d-3c21220bf425.log
	2015-07-21 18:53:40,254 INFO [main]: log.PerfLogger (PerfLogger.java:PerfLogBegin(108)) - <PERFLOG method=deserializePlan from=org.apache.hadoop.hive.ql.exec.Utilities>
	2015-07-21 18:53:40,255 INFO [main]: exec.Utilities (Utilities.java:deserializePlan(819)) - Deserializing MapredWork via kryo
	2015-07-21 18:53:40,590 INFO [main]: log.PerfLogger (PerfLogger.java:PerfLogEnd(135)) - </PERFLOG method=deserializePlan start=1437485020254 end=1437485020590 duration=336 from=org.apache.hadoop.hive.ql.exec.Utilities>
	2015-07-21 18:53:40,625 INFO [main]: mr.ExecDriver (ExecDriver.java:execute(285)) - Using org.apache.hadoop.hive.ql.io.CombineH

prongs / lens_demo_commands

Created July 13, 2015 06:21

lens demo commands

	showterm
	cd ~/Git/incubator-lens
	export PS1="\u@\h:\w [$?]\n\$"
	# Now directory is on one line, command on next
	# compile: Not running right now. http://showterm.io/48a63d963385a442e9978
	# mvn clean install -DskipTests -Dcheckstyle.skip -Dfindbugs.skip=true -DskipCheck
	cd lens-dist/target/bin/bin/
	echo $HADOOP_HOME
	echo $HIVE_HOME
	ls

prongs / gist:b2cd03babf93da94001c

Created June 26, 2015 09:06

{hive.server.tcp.keepalive=true, lens.query.enable.mail.notify=false, lens.server.statistics.warehouse.dir=hdfs://192.168.138.164:8020/user/hive/warehouse, lens.server.drivers=org.apache.lens.driver.hive.HiveDriver,org.apache.lens.driver.jdbc.JDBCDriver, hive.server2.log.redirection.enabled=false, hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat, lens.query.output.enable.compression=true, mapred.job.name=60dd690f-2365-43b7-96d6-2e03da8b9a3a, lens.query.hdfs.output.path=hdfsout, lens.server.metastore.service.impl=org.apache.lens.server.metastore.CubeMetastoreServiceImpl, lens.cube.query.fail.if.data.partial=false, lens.server.user.resolver.propertybased.filename=/path/to/propertyfile, hive.metastore.batch.retrieve.table.partition.max=500, lens.query.result.size.format.threshold=10737418240, lens.server.user.resolver.ldap.search.filter=(&(objectClass=user)(sAMAccountName=%s)), lens.server.servicenames=session,query,metastore,scheduler,quota, lens.server.serverMode.ws.filter.impl=org.apache.lens.se

prongs / gist:a74b456e27547127efcb

Created June 22, 2015 08:27

	def f(a):
	if a is full file path:
	return a
	elif a is a full directory path:
	ret = []
	for line in readfile(a+orderfile):
	if line starts with "/" or has ":/":
	ret += f(line)
	else: # relative path
	ret += f(a+"/"+line)

prongs / gist:66f56b7e08f2fee62c5f

Created May 26, 2015 10:15

	26 May 2015 10:06:01,191 [estimate-23] INFO org.apache.lens.cube.parse.CandidateTableResolver - Not considering fact table:downloadltv_raw as column processed_hour_id is not available
	26 May 2015 10:06:01,192 [estimate-23] INFO org.apache.lens.cube.parse.CandidateTableResolver - Not considering fact table:downloadltv_raw as columns [site_channel_id] are not available
	26 May 2015 10:06:01,192 [estimate-23] INFO org.apache.lens.cube.parse.CandidateTableResolver - Not considering fact table:rrcube_raw as column download_time is not available
	26 May 2015 10:06:01,192 [estimate-23] INFO org.apache.lens.cube.parse.CandidateTableResolver - Not considering fact table:rrcube_raw as columns [ltv_install, ltv_event_count, ltv_event_value, ltv_impression, dl_confident_install, ltv_click, ltv_valid_click] is not available
	26 May 2015 10:06:01,192 [estimate-23] INFO org.apache.lens.cube.parse.CandidateTableResolver - Not considering fact table:day_agg1_demand_fact as column download_time is not available
	26 May 2

prongs / gist:5ede38fe973f5794c3fc

Created May 26, 2015 09:09


	Logged in as: dr.who
	Application
	About
	Jobs
	Tools
	Log Type: syslog
	Log Upload Time: 26-May-2015 08:54:28
	Log Length: 113846
	2015-05-26 08:52:17,696 INFO [main] org.apache.hadoop.metrics2.impl.MetricsConfig: loaded properties from hadoop-metrics2.properties

prongs / prime_path.py

Created May 16, 2015 18:39

	class Node:
	Node left
	Node right
	int data
	int prime_depth # extra space
	int path_size # extra space

	global max_path_size = -1
	global max_path_node = None

Rajat Khandelwal prongs