Last active
April 22, 2019 17:45
-
-
Save jameswinegar/1c2409996ed739c2b14e54af358d3bad to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "import pyspark" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "from pyspark.sql import SparkSession\n", | |
| "app_name = \"wk12_demo\"\n", | |
| "master = \"local[*]\"\n", | |
| "spark = SparkSession\\\n", | |
| " .builder\\\n", | |
| " .appName(app_name)\\\n", | |
| " .master(master)\\\n", | |
| " .getOrCreate()\n", | |
| "sc = spark.sparkContext" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "\n", | |
| " <div>\n", | |
| " <p><b>SparkSession - in-memory</b></p>\n", | |
| " \n", | |
| " <div>\n", | |
| " <p><b>SparkContext</b></p>\n", | |
| "\n", | |
| " <p><a href=\"http://jupyter-winegarj:4040\">Spark UI</a></p>\n", | |
| "\n", | |
| " <dl>\n", | |
| " <dt>Version</dt>\n", | |
| " <dd><code>v2.4.1</code></dd>\n", | |
| " <dt>Master</dt>\n", | |
| " <dd><code>local[*]</code></dd>\n", | |
| " <dt>AppName</dt>\n", | |
| " <dd><code>wk12_demo</code></dd>\n", | |
| " </dl>\n", | |
| " </div>\n", | |
| " \n", | |
| " </div>\n", | |
| " " | |
| ], | |
| "text/plain": [ | |
| "<pyspark.sql.session.SparkSession at 0x7f1c281ccef0>" | |
| ] | |
| }, | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "spark" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 7, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "6" | |
| ] | |
| }, | |
| "execution_count": 7, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "rdd = sc.parallelize([1,2,3])\n", | |
| "rdd.sum()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.7.1" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment