groceryheist/cdsc_reddit
1
0
Fork 0
Code Activity

update examples with working streaming

Browse Source
This commit is contained in:
Nate E TeBlunthuis
2020-07-07 11:47:17 -07:00
parent 40d4563770
commit e22ddf23da
2 changed files with 23 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/pyarrow_reading.py
View File

@@ -1,8 +1,8 @@
import pyarrow.dataset as ds
import pyarrow as pa
# A pyarrow dataset abstracts reading, writing, or filtering a parquet file. It does not read dataa into memory.
#dataset = ds.dataset(pathlib.Path('/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet/'), format='parquet', partitioning='hive')
dataset = ds.dataset('/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet/', format='parquet', partitioning='hive')
dataset = ds.dataset('/gscratch/comdata/output/reddit_comments_by_subreddit.parquet/', format='parquet')
# let's get all the comments to two subreddits:
subreddits_to_pull = ['seattle','seattlewa']