###Tested with:
- Spark 2.0.0 pre-built for Hadoop 2.7
- Mac OS X 10.11
- Python 3.5.2
Use s3 within pyspark with minimal hassle.
| import requests | |
| import time | |
| import os | |
| import sys | |
| import openai | |
| import tiktoken | |
| from termcolor import colored | |
| openai.api_key = open(os.path.expanduser('~/.openai')).read().strip() |
| object UserEntityModule { | |
| case class UserEntityF[F[_], G[_]](id: G[Option[Long]] = None, username: F[String], password: F[String]) | |
| type Id[A] = A | |
| type Forget[A] = Unit | |
| // You can also just use Option if you don't care | |
| // for the domain-specific type | |
| sealed trait Updatable[+A] { | |
| def foreach(f : A => Unit): Unit = this match { |
| ## ~/.i3status.conf | |
| # i3status configuration file. | |
| # see "man i3status" for documentation. | |
| # It is important that this file is edited as UTF-8. | |
| # The following line should contain a sharp s: | |
| # ß | |
| # If the above line is not correctly displayed, fix your editor first! |
| ## ~/.i3status.conf | |
| # i3status configuration file. | |
| # see "man i3status" for documentation. | |
| # It is important that this file is edited as UTF-8. | |
| # The following line should contain a sharp s: | |
| # ß | |
| # If the above line is not correctly displayed, fix your editor first! |
| object StaticFile { | |
| // Various necessary imports. Notes: | |
| // | |
| // 1. fs2 is necessary. See https://github.com/functional-streams-for-scala/fs2 | |
| // 2. streamz is necessary. See https://github.com/krasserm/streamz | |
| // 3. Apache Tika is used to infer MIME types from file names, because it's more reliable and | |
| // fully-featured than using java.nio.file.Files.probeContentType(). | |
| // | |
| // If using SBT, you'll want these library dependencies and resolvers: |
| // Inspired by a tweet from @trautonen 1/13/2016 | |
| // Use Source.unfoldAsync to turn paginated database results into an akka-streams Source | |
| // unfold is the inverse of fold | |
| case class Page[T](pageNumber:Long, totalPages:Long, contents:List[T]) | |
| case class Thing(id: Long, name: String = "foo") | |
| val totalPages = 5 // | |
| val pageSize = 3 |