Tincu Gabriel gabriel-tincu

Use this to crawl romanian news sites and store content to either mongo or ES

 virtualenv venv --no-site-packages --python python3
 source vev/bin/activate
 pip install -r requirements.txt
 python main.py

	@Test
	public void testListPartitionReassignmentsResponse() {
	List<ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics> topList = new ArrayList<>();
	ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics top = new ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics().setName("fooTopic");
	topList.add(top);
	List<Integer> parts = new ArrayList<>();
	parts.add(10);
	top.setPartitionIndexes(parts);
	ListPartitionReassignmentsRequest r = new ListPartitionReassignmentsRequest.Builder(new ListPartitionReassignmentsRequestData().setTimeoutMs(10000).setTopics(topList)).build();
	ByteBuffer bb = r.serialize(new RequestHeader(ApiKeys.LIST_PARTITION_REASSIGNMENTS, (short) 0, "foo-client", (short) 0));

	import socketserver
	import socket
	import time
	import logging
	headers = "HTTP/1.1 200 OK\nContent-Type: text/html\n\n"
	payload = "sleep 1\n{}\nsleep 1\n".format("\n".join(["echo foo"]*20))
	more = "echo final lines"

	logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
	logger = logging.getLogger('server')

	import unittest

	class MyTest(unittest.TestCase):
	def testFoo(self):
	self.assertEqual(3,3)

	def testBar(self):
	self.assertTrue(False)

	import unittest, time

	class MyResult(unittest.TestResult):
	def __init__(self, stream=None, descriptions=None, verbosity=None):
	super(MyResult, self).__init__(stream=None, descriptions=None, verbosity=None)
	self.data = {}

	def startTest(self, test):
	print(test, type(test))
	super(MyResult, self).startTest(test)

	local kube = import "../lib/kube.libsonnet";

	local hardCodedEnv = {
	ES_BACKUP_BUCKET: "zageno_elasticsearch_backup",
	ES_CLIENT_TIMEOUT: "53",
	SALESFORCE_PULL_ACTIVE: "1",
	SALESFORCE_LIVE: "1",
	SALESFORCE_ACTIVE: "1",
	STATIC_URL: "/static/",
	ES_SNAPSHOT_NAME: "gcs_backup",

	client:
	hosts:
	${ES_HOST}
	timeout: 1800
	---
	actions:
	1:
	action: shrink
	description: shrink them indices
	options:

	#!/usr/bin/env bash
	cd $(dirname $0)
	root_path=$(dirname $(dirname $(pwd -P)))
	main_re="#[[:space:]]+TODO[[:space:]]+\[[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2}\]"
	date_re="[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2}"
	today=$(date +%s)
	search_path=$root_path
	OIFS=$IFS
	found_expired=