dashaun · March 29, 2021 21:09
diff --git a/docker-compose.yml b/docker-compose.yml
 version: "2"
 services:
  tweets-search:
    image: abhirockzz/redisearch-tweets-search
    ports:
      - 80:80
    environment:
      - REDIS_HOST=<azure redis host name>
      - REDIS_PASSWORD=<azure redis access key>
      - REDISEARCH_INDEX_NAME=tweets-index
  tweets-indexer:
    image: abhirockzz/redisearch-tweets-consumer
    environment:
      - TWITTER_CONSUMER_KEY=<twitter api consumer key>
      - TWITTER_CONSUMER_SECRET_KEY=<twitter api consumer secret>
      - TWITTER_ACCESS_TOKEN=<twitter api access token>
      - TWITTER_ACCESS_SECRET_TOKEN=<twitter api access secret>
      - REDIS_HOST=<azure redis host name>
      - REDIS_PASSWORD=<azure redis access key>
      - REDISEARCH_INDEX_NAME=tweets-index
diff --git a/gistfile1.txt b/gistfile1.txt
 docker login azure 
 docker context create aci aci-context 
 docker context use aci-context 
diff --git a/gistfile10.txt b/gistfile10.txt
 HTTP/1.1 200 OK 
 Page-Size: 10 
 Search-Hits: 12 
 Date: Mon, 25 Jan 2021 13:21:52 GMT 
 Content-Type: text/plain; charset=utf-8 
 Transfer-Encoding: chunked 
 //JSON array of documents (omitted) 
diff --git a/gistfile11.txt b/gistfile11.txt
 curl -i "$REDISEARCH_API_BASE_URL?q=*&offset_limit=0,100" 

 offset_limit=0,100 will return up to 100 documents ( limit ) starting with the first one ( offset = 0). 
diff --git a/gistfile12.txt b/gistfile12.txt
 curl -i "$REDISEARCH_API_BASE_URL?q=@source:iphone" 
diff --git a/gistfile13.txt b/gistfile13.txt
 curl -i "$REDISEARCH_API_BASE_URL?q=@location:india&fields=user,text" 
diff --git a/gistfile14.txt b/gistfile14.txt
 curl -i "$REDISEARCH_API_BASE_URL?q=@user:jo*" 
diff --git a/gistfile15.txt b/gistfile15.txt
 bash curl -i $REDISEARCH_API_BASE_URL?q=@location:India @source:android
diff --git a/gistfile16.txt b/gistfile16.txt
 curl -i "$REDISEARCH_API_BASE_URL?q=@hashtags:\{potus|cov*\}" 
diff --git a/gistfile17.txt b/gistfile17.txt
 curl -i "$REDISEARCH_API_BASE_URL?q=@hashtags:{biden} @created:[1611556920000000000 1711556930000000000]"
diff --git a/gistfile18.txt b/gistfile18.txt
 curl -i "$REDISEARCH_API_BASE_URL?q=*&fields=coordinates" 
 curl -i "$REDISEARCH_API_BASE_URL?q=@coordinates:[-122.41 37.77 10 km]" 
diff --git a/gistfile19.txt b/gistfile19.txt
 docker compose down -p azure-redisearch-app 
diff --git a/gistfile2.txt b/gistfile2.txt
 git clone https://github.com/abhirockzz/redisearch-tweet-analysis 
 cd redisearch-tweet-analysis
diff --git a/gistfile20.txt b/gistfile20.txt
 config := oauth1.NewConfig(GetEnvOrFail(consumerKeyEnvVar), GetEnvOrFail(consumerSecretKeyEnvVar)) 
 token := oauth1.NewToken(GetEnvOrFail(accessTokenEnvVar), GetEnvOrFail(accessSecretEnvVar)) 
 httpClient := config.Client(oauth1.NoContext, token) 
 client := twitter.NewClient(httpClient) 
diff --git a/gistfile21.txt b/gistfile21.txt
 demux := twitter.NewSwitchDemux() 
 demux.Tweet = func(tweet *twitter.Tweet) { 
  if !tweet.PossiblySensitive { 
    go index.AddData(tweetToMap(tweet)) 
    time.Sleep(3 * time.Second) 
  } 
 } 
 go func() { 
  for tweet := range stream.Messages { 
    demux.Handle(tweet) 
  } 
 }() 
diff --git a/gistfile22.txt b/gistfile22.txt
 host := GetEnvOrFail(redisHost) 
 password := GetEnvOrFail(redisPassword) 
 indexName = GetEnvOrFail(indexNameEnvVar) 
 pool = &redis.Pool{Dial: func() (redis.Conn, error) { 
  return redis.Dial("tcp", host, redis.DialPassword(password), redis.DialUseTLS(true), redis.DialTLSConfig(&tls.Config{MinVersion: tls}
  } 
diff --git a/gistfile23.txt b/gistfile23.txt
 rsClient := redisearch.NewClientFromPool(pool, indexName) 
 err := rsClient.DropIndex(true) 
 schema := redisearch.NewSchema(redisearch.DefaultOptions). 
 AddField(redisearch.NewTextFieldOptions("id", redisearch.TextFieldOptions{})). 
 AddField(redisearch.NewTextFieldOptions("user", redisearch.TextFieldOptions{})). 
 AddField(redisearch.NewTextFieldOptions("text", redisearch.TextFieldOptions{})). 
 AddField(redisearch.NewTextFieldOptions("source", redisearch.TextFieldOptions{})). 
 //tags are comma-separated by default 
 AddField(redisearch.NewTagFieldOptions("hashtags", redisearch.TagFieldOptions{})). 
 AddField(redisearch.NewTextFieldOptions("location", redisearch.TextFieldOptions{})). 
 AddField(redisearch.NewNumericFieldOptions("created", redisearch.NumericFieldOptions{Sortable: true})). 
 AddField(redisearch.NewGeoFieldOptions("coordinates", redisearch.GeoFieldOptions{})) 
 indexDefinition := redisearch.NewIndexDefinition().AddPrefix(indexDefinitionHashPrefix) 
 err = rsClient.CreateIndexWithIndexDefinition(schema, indexDefinition) 
diff --git a/gistfile24.txt b/gistfile24.txt
 func AddData(tweetData map[string]interface{}) { 
 conn := pool.Get() 
 hashName := fmt.Sprintf("tweet:%s", tweetData["id"]) 
 val := redis.Args{hashName}.AddFlat(tweetData) 
 _, err := conn.Do("HSET", val...) 
 } 
diff --git a/gistfile25.txt b/gistfile25.txt
 qParams, err := url.ParseQuery(req.URL.RawQuery) 
 if err != nil { 
 log.Println("invalid query params") 
 http.Error(rw, err.Error(), http.StatusBadRequest) 
 return 
 } 
 searchQuery := qParams.Get(queryParamQuery) 
 query := redisearch.NewQuery(searchQuery) 
diff --git a/gistfile26.txt b/gistfile26.txt
 http://localhost:8080/search?q=@source:Web&fields=user,source&offset_limit=5,100 
 fields := qParams.Get(queryParamFields) 
 offsetAndLimit := qParams.Get(queryParamOffsetLimit) 
diff --git a/gistfile27.txt b/gistfile27.txt
 docs, total, err := rsClient.Search(query) 
 response := []map[string]interface{}{} 
 for _, doc := range docs { 
 response = append(response, doc.Properties) 
 } 
 rw.Header().Add(responseHeaderSearchHits, strconv.Itoa(total)) 
 err = json.NewEncoder(rw).Encode(response) 
diff --git a/gistfile3.txt b/gistfile3.txt
 docker compose up -p azure-redisearch-app 
diff --git a/gistfile4.txt b/gistfile4.txt
 [+] Running 1/3 
 ⠿ Group azure-redisearch-app Created 8.3s ⠸ tweets-search Creating 6.3s ⠸ tweets-indexer Creating 6.3s 
diff --git a/gistfile5.txt b/gistfile5.txt
 docker logs azure-redisearch-app_tweets-indexer 
 docker logs azure-redisearch-app_tweets-search 
diff --git a/gistfile6.txt b/gistfile6.txt
 docker ps 
 //output 
 CONTAINER ID IMAGE COMMAND STATUS PORTS azure-redisearch-app_tweets-search abhirockzz/redisearch-tweets-search Running 20.197.96.54:80->80/tcazure-redisearch-app_tweets-indexer abhirockzz/redisearch-tweets-consumer Running 
diff --git a/gistfile7.txt b/gistfile7.txt
 id - this is a the Tweet ID ( TEXT attribute) 
 user - the is the screen name ( TEXT attribute) 
 text - tweet contents ( TEXT attribute) 
 source - tweet source e.g. Twitter for Android, Twitter Web App, Twitter for iPhone ( TEXT attribute) 
 hashtags - hashtags (if any) in the tweet (available in CSV format as a TAG attribute) 
 location - tweet location (if available). this is a user defined location (not the exact location per se) 
 created - timestamp (epoch) of the tweet. this is NUMERIC field and can be used for range queries 
 coordinates - geographic location (longitude, latitude) if made available by the client ( GEO attribute) 
diff --git a/gistfile8.txt b/gistfile8.txt
 export REDISEARCH_API_BASE_URL=<for example, http://20.197.96.54:80/search> 
diff --git a/gistfile9.txt b/gistfile9.txt
 curl -i $REDISEARCH_API_BASE_URL?q=*
	version: "2"
	services:
	tweets-search:
	image: abhirockzz/redisearch-tweets-search
	ports:
	- 80:80
	environment:
	- REDIS_HOST=<azure redis host name>
	- REDIS_PASSWORD=<azure redis access key>
	- REDISEARCH_INDEX_NAME=tweets-index
	tweets-indexer:
	image: abhirockzz/redisearch-tweets-consumer
	environment:
	- TWITTER_CONSUMER_KEY=<twitter api consumer key>
	- TWITTER_CONSUMER_SECRET_KEY=<twitter api consumer secret>
	- TWITTER_ACCESS_TOKEN=<twitter api access token>
	- TWITTER_ACCESS_SECRET_TOKEN=<twitter api access secret>
	- REDIS_HOST=<azure redis host name>
	- REDIS_PASSWORD=<azure redis access key>
	- REDISEARCH_INDEX_NAME=tweets-index
	docker login azure
	docker context create aci aci-context
	docker context use aci-context
	HTTP/1.1 200 OK
	Page-Size: 10
	Search-Hits: 12
	Date: Mon, 25 Jan 2021 13:21:52 GMT
	Content-Type: text/plain; charset=utf-8
	Transfer-Encoding: chunked
	//JSON array of documents (omitted)
	curl -i "$REDISEARCH_API_BASE_URL?q=*&offset_limit=0,100"

	offset_limit=0,100 will return up to 100 documents ( limit ) starting with the first one ( offset = 0).
	curl -i "$REDISEARCH_API_BASE_URL?q=*&fields=coordinates"
	curl -i "$REDISEARCH_API_BASE_URL?q=@coordinates:[-122.41 37.77 10 km]"
	git clone https://github.com/abhirockzz/redisearch-tweet-analysis
	cd redisearch-tweet-analysis
	config := oauth1.NewConfig(GetEnvOrFail(consumerKeyEnvVar), GetEnvOrFail(consumerSecretKeyEnvVar))
	token := oauth1.NewToken(GetEnvOrFail(accessTokenEnvVar), GetEnvOrFail(accessSecretEnvVar))
	httpClient := config.Client(oauth1.NoContext, token)
	client := twitter.NewClient(httpClient)
	demux := twitter.NewSwitchDemux()
	demux.Tweet = func(tweet *twitter.Tweet) {
	if !tweet.PossiblySensitive {
	go index.AddData(tweetToMap(tweet))
	time.Sleep(3 * time.Second)
	}
	}
	go func() {
	for tweet := range stream.Messages {
	demux.Handle(tweet)
	}
	}()
	host := GetEnvOrFail(redisHost)
	password := GetEnvOrFail(redisPassword)
	indexName = GetEnvOrFail(indexNameEnvVar)
	pool = &redis.Pool{Dial: func() (redis.Conn, error) {
	return redis.Dial("tcp", host, redis.DialPassword(password), redis.DialUseTLS(true), redis.DialTLSConfig(&tls.Config{MinVersion: tls}
	}
	rsClient := redisearch.NewClientFromPool(pool, indexName)
	err := rsClient.DropIndex(true)
	schema := redisearch.NewSchema(redisearch.DefaultOptions).
	AddField(redisearch.NewTextFieldOptions("id", redisearch.TextFieldOptions{})).
	AddField(redisearch.NewTextFieldOptions("user", redisearch.TextFieldOptions{})).
	AddField(redisearch.NewTextFieldOptions("text", redisearch.TextFieldOptions{})).
	AddField(redisearch.NewTextFieldOptions("source", redisearch.TextFieldOptions{})).
	//tags are comma-separated by default
	AddField(redisearch.NewTagFieldOptions("hashtags", redisearch.TagFieldOptions{})).
	AddField(redisearch.NewTextFieldOptions("location", redisearch.TextFieldOptions{})).
	AddField(redisearch.NewNumericFieldOptions("created", redisearch.NumericFieldOptions{Sortable: true})).
	AddField(redisearch.NewGeoFieldOptions("coordinates", redisearch.GeoFieldOptions{}))
	indexDefinition := redisearch.NewIndexDefinition().AddPrefix(indexDefinitionHashPrefix)
	err = rsClient.CreateIndexWithIndexDefinition(schema, indexDefinition)