Skip to content

Instantly share code, notes, and snippets.

@pinei
Created January 11, 2024 00:04
Show Gist options
  • Select an option

  • Save pinei/4e93cf7805569c2e16ff631c03c21ba5 to your computer and use it in GitHub Desktop.

Select an option

Save pinei/4e93cf7805569c2e16ff631c03c21ba5 to your computer and use it in GitHub Desktop.
Bluesky Firehose

Firehose

Usar o stream do Bluesky para captura de posts

Ref.: https://atproto.com/specs/event-stream

Exemplo de post capturado:

{'record':
	{'embed':
		{'external':
			{'title': 'Divided by Politics, a Colorado Town Mends Its Broken Bones',
			 'uri': 'https://www.nytimes.com/2023/11/30/us/politics/silverton-division.html',
			 'thumb':
			 	{'mimeType': 'image/jpeg', 'size': 626157, '$type': 'blob',
			 	'ref': 'bafkreifvmpjrmoaz4ik2rdbl63uqs55pe4tn2l2ydewu4blmjlchczq7eq'},
		 	 'description': 'Two years after death threats and aspersions roiled little Silverton, the town has found a semblance of peace and a lesson for a ruptured nation.'
		 	},
		 	'$type': 'app.bsky.embed.external'
	 	},
	 	'langs': ['en'],
	 	'$type': 'app.bsky.feed.post',
	 	'text':
	 		'On town-planning as an opportunity for municipality-wide mediation + conflict management',
	 		'createdAt': '2023-12-04T03:32:55.198Z'},
	'uri': 'at://did:plc:jtzxnbwxwbywld23yvnyokzf/app.bsky.feed.post/3kfov52ovpa22',
	'cid': 'bafyreia7sqf2t7mioz3qr4l2ivddtzbywbg5m5lfs46645fwhxioagrksa',
	'author': 'did:plc:jtzxnbwxwbywld23yvnyokzf'
}

O atributo record pode ser de 2 tipos:

  • atproto.xrpc_client.models.app.bsky.feed.post.Main
  • atproto.xrpc_client.models.dot_dict.DotDict

Exemplo com DotDict

{
  'record': {
    'text': 'Animal Crossing! Mao Mao\n\nI drew this for fun. I thought it would cute to have a little bit of a crossover with Animal Crossing and MMHOPH.\n\nAnimal Crossing (C) of Nintendo \n\nMao Mao (C) of Cartoonnetwork and Parker Simmons\n\n#animalcrossing #crossover #maomao #nintendo #mmhoph #fanart #myart',
    'embed': {
      '$type': 'app.bsky.embed.images',
         'images': [
        {'alt': '', 'image': {'mimeType': 'image/jpeg', '$type': 'blob', 'ref': 'bafkreia5smsetguxvxaw4pnn4q2fiwpqgeod7w77he4fnnsxc2g2syr7ea', 'size': 374537
          }, 'aspectRatio': {'height': 1200, 'width': 900
          }
        }
      ]
    }, 'facets': [
      {'index': {'byteStart': 225, 'byteEnd': 240
        }, 'features': [
          {'$type': 'app.bsky.richtext.facet#tag', 'tag': 'animalcrossing'
          }
        ]
      },
      {'index': {'byteStart': 241, 'byteEnd': 251
        }, 'features': [
          {'$type': 'app.bsky.richtext.facet#tag', 'tag': 'crossover'
          }
        ]
      },
      {'features': [
          {'tag': 'maomao', '$type': 'app.bsky.richtext.facet#tag'
          }
        ], 'index': {'byteEnd': 259, 'byteStart': 252
        }
      },
      {'features': [
          {'tag': 'nintendo', '$type': 'app.bsky.richtext.facet#tag'
          }
        ], 'index': {'byteStart': 260, 'byteEnd': 269
        }
      },
      {'features': [
          {'$type': 'app.bsky.richtext.facet#tag', 'tag': 'mmhoph'
          }
        ], 'index': {'byteStart': 270, 'byteEnd': 277
        }
      },
      {'index': {'byteEnd': 285, 'byteStart': 278
        }, 'features': [
          {'$type': 'app.bsky.richtext.facet#tag', 'tag': 'fanart'
          }
        ]
      },
      {'index': {'byteStart': 286, 'byteEnd': 292
        }, 'features': [
          {'tag': 'myart', '$type': 'app.bsky.richtext.facet#tag'
          }
        ]
      }
    ], 'createdAt': '2023-12-04T03: 46: 03.052Z', 'langs': ['en'
    ], '$type': 'app.bsky.feed.post'
  }, 'uri': 'at: //did:plc:rsitlvl23yniizd2h35vuaa7/app.bsky.feed.post/3kfovuhnu452f', 'cid': 'bafyreihl4sk72gj2fchitpkvxnkmucast5chgg5o3pjwdg74h2mkis3nuq', 'author': 'did:plc:rsitlvl23yniizd2h35vuaa7'}

Exemplo com Main

from atproto import CAR, AtUri
from atproto.firehose import FirehoseSubscribeReposClient, parse_subscribe_repos_message
from atproto.firehose.models import MessageFrame
from atproto.xrpc_client import models
from atproto.xrpc_client.models import get_or_create, ids, is_record_type
client = FirehoseSubscribeReposClient()
def process_commit(commit):
car = CAR.from_bytes(commit.blocks)
for op in commit.ops:
uri = AtUri.from_str(f'at://{commit.repo}/{op.path}')
if op.action == 'create':
if not op.cid:
continue
create_info = {'uri': str(uri), 'cid': str(op.cid), 'author': commit.repo}
record_raw_data = car.blocks.get(op.cid)
if not record_raw_data:
continue
record = get_or_create(record_raw_data, strict=False)
if uri.collection == ids.AppBskyFeedPost and is_record_type(record, ids.AppBskyFeedPost):
print(type(record))
print({'record': record, **create_info})
def on_message_handler(message) -> None:
commit = parse_subscribe_repos_message(message)
# we need to be sure that it's commit message with .blocks inside
if not isinstance(commit, models.ComAtprotoSyncSubscribeRepos.Commit):
return
process_commit(commit)
client.start(on_message_handler)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment