Laboratory 3 - Stream Data producing

import re
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
sc = spark.sparkContext

sc.textFile("RDD_input")\
.map(lambda x: re.findall(r"[a-z']+", x.lower()))\
.flatMap(lambda x: [(y, 1) for y in x])\
.reduceByKey(lambda x, y: x+y)\
.collect()

from socket import *
import time

rdd = list()
# example how to create list of events from text file 
with open("RDD_input", 'r') as ad: 
    for line in ad:
        rdd.append(line)
        

HOST = 'localhost'
PORT = 9998
ADDR = (HOST, PORT)
tcpSock = socket(AF_INET, SOCK_STREAM)
tcpSock.bind(ADDR) 
tcpSock.listen(5)

# main loop
while True:
    c, addr = tcpSock.accept()
    print("got connection")
    for line in rdd:
        try:
            c.send(line.encode())
            time.sleep(1)
        except:
            break
    c.close()
    print("disconnected") 

import re 
from pyspark import SparkContext
from pyspark.streaming import StreamingContext

words = lines.flatMap(lambda x: re.findall(r"[a-z']+", x.lower()))
wordCounts = words.map(lambda word: (word,1)).reduceByKey(lambda x,y: x+y)

wordCounts.pprint()

ssc.start()             # Start the computation
ssc.awaitTermination()  # Wait for the computation to terminate
ssc.stop(True,True)