Skip to content

Instantly share code, notes, and snippets.

@reddgr

reddgr/search_console_api.ipynb

Created January 29, 2025 11:41
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save reddgr/2be4d158e7568f4788138e038ca7cb51 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save reddgr/2be4d158e7568f4788138e038ca7cb51 to your computer and use it in GitHub Desktop.
Download ZIP
Google Search Console API usage in Python
Display the source blob
Display the rendered blob
Raw
search_console_api.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 48,
"metadata": {},
"outputs": [],
"source": [
"import argparse\n",
"import sys\n",
"from googleapiclient import sample_tools\n",
"from googleapiclient.discovery import build\n",
"from googleapiclient.errors import HttpError\n",
"import os\n",
"import pandas as pd\n",
"from bs4 import BeautifulSoup\n",
"import requests\n",
"from tqdm import tqdm\n",
"import random\n",
"from IPython.display import clear_output\n",
"\n",
"from dotenv import load_dotenv\n",
"load_dotenv(\"../../../../../../apis/.env\")\n",
"client_secrets_path = os.getenv(\"GSC_CLIENT_SECRETS_PATH\")"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Authenticated user's verified properties:\n",
"Property: sc-domain:talkingtochatbots.com, Permission Level: siteOwner\n",
"Property: sc-domain:reddgr.com, Permission Level: siteOwner\n",
"Property: sc-domain:asocall.blogspot.com, Permission Level: siteOwner\n"
]
}
],
"source": [
"# Initialize the API service\n",
"service, flags = sample_tools.init(\n",
" [],\n",
" \"searchconsole\",\n",
" \"v1\",\n",
" __doc__,\n",
" client_secrets_path,\n",
" scope=\"https://www.googleapis.com/auth/webmasters.readonly\",\n",
")\n",
"clear_output(wait=True)\n",
"\n",
"# Verify authenticated user's permissions\n",
"about_info = service.sites().list().execute()\n",
"print(\"Authenticated user's verified properties:\")\n",
"\n",
"# Print properties and permission levels for verification\n",
"if \"siteEntry\" in about_info:\n",
" for site in about_info[\"siteEntry\"]:\n",
" print(f\"Property: {site['siteUrl']}, Permission Level: {site['permissionLevel']}\")\n",
"else:\n",
" print(\"No verified properties found.\")"
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Available dates data:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>clicks</th>\n",
" <th>impressions</th>\n",
" <th>ctr</th>\n",
" <th>position</th>\n",
" <th>date</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2</td>\n",
" <td>206</td>\n",
" <td>0.009709</td>\n",
" <td>20.029126</td>\n",
" <td>2023-11-20</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>242</td>\n",
" <td>0.004132</td>\n",
" <td>21.297521</td>\n",
" <td>2023-11-21</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0</td>\n",
" <td>239</td>\n",
" <td>0.000000</td>\n",
" <td>19.778243</td>\n",
" <td>2023-11-22</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2</td>\n",
" <td>233</td>\n",
" <td>0.008584</td>\n",
" <td>20.120172</td>\n",
" <td>2023-11-23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2</td>\n",
" <td>197</td>\n",
" <td>0.010152</td>\n",
" <td>20.350254</td>\n",
" <td>2023-11-24</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>2</td>\n",
" <td>195</td>\n",
" <td>0.010256</td>\n",
" <td>17.148718</td>\n",
" <td>2023-11-25</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>1</td>\n",
" <td>206</td>\n",
" <td>0.004854</td>\n",
" <td>20.786408</td>\n",
" <td>2023-11-26</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>1</td>\n",
" <td>213</td>\n",
" <td>0.004695</td>\n",
" <td>16.281690</td>\n",
" <td>2023-11-27</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>7</td>\n",
" <td>1040</td>\n",
" <td>0.006731</td>\n",
" <td>10.092308</td>\n",
" <td>2023-11-28</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>6</td>\n",
" <td>550</td>\n",
" <td>0.010909</td>\n",
" <td>11.767273</td>\n",
" <td>2023-11-29</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>3</td>\n",
" <td>245</td>\n",
" <td>0.012245</td>\n",
" <td>15.310204</td>\n",
" <td>2023-11-30</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" clicks impressions ctr position date\n",
"0 2 206 0.009709 20.029126 2023-11-20\n",
"1 1 242 0.004132 21.297521 2023-11-21\n",
"2 0 239 0.000000 19.778243 2023-11-22\n",
"3 2 233 0.008584 20.120172 2023-11-23\n",
"4 2 197 0.010152 20.350254 2023-11-24\n",
"5 2 195 0.010256 17.148718 2023-11-25\n",
"6 1 206 0.004854 20.786408 2023-11-26\n",
"7 1 213 0.004695 16.281690 2023-11-27\n",
"8 7 1040 0.006731 10.092308 2023-11-28\n",
"9 6 550 0.010909 11.767273 2023-11-29\n",
"10 3 245 0.012245 15.310204 2023-11-30"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"property_uri = 'sc-domain:reddgr.com' # Replace with your property URI\n",
"start_date = '2023-11-20' # Replace with your start date in YYYY-MM-DD format\n",
"end_date = '2023-11-30' # Replace with your end date in YYYY-MM-DD format\n",
"# Build the request to check available dates\n",
"request = {\n",
" \"startDate\": start_date,\n",
" \"endDate\": end_date,\n",
" \"dimensions\": [\"date\"],\n",
"}\n",
"\n",
"response = service.searchanalytics().query(siteUrl=property_uri, body=request).execute()\n",
"\n",
"# Convert the response to a Pandas DataFrame\n",
"if \"rows\" in response:\n",
" data = response[\"rows\"]\n",
" df = pd.DataFrame(data)\n",
" if \"keys\" in df.columns:\n",
" df[\"date\"] = df[\"keys\"].apply(lambda x: x[0] if x else None)\n",
" df.drop(\"keys\", axis=1, inplace=True)\n",
"else:\n",
" df = pd.DataFrame(columns=[\"date\", \"clicks\", \"impressions\", \"ctr\", \"position\"])\n",
"\n",
"# Display the DataFrame\n",
"print(\"Available dates data:\")\n",
"display(df)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## URL inspection"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'inspectionResult': {'inspectionResultLink': 'https://search.google.com/search-console/inspect?resource_id=sc-domain:talkingtochatbots.com&id=gqOlQrFkOo42X_cPF_b4DA&utm_medium=link&utm_source=api', 'indexStatusResult': {'verdict': 'NEUTRAL', 'coverageState': 'URL is unknown to Google', 'robotsTxtState': 'ROBOTS_TXT_STATE_UNSPECIFIED', 'indexingState': 'INDEXING_STATE_UNSPECIFIED', 'pageFetchState': 'PAGE_FETCH_STATE_UNSPECIFIED'}, 'mobileUsabilityResult': {'verdict': 'VERDICT_UNSPECIFIED'}}}\n",
"verdict: NEUTRAL\n",
"coverageState: URL is unknown to Google\n",
"robotsTxtState: ROBOTS_TXT_STATE_UNSPECIFIED\n",
"indexingState: INDEXING_STATE_UNSPECIFIED\n",
"pageFetchState: PAGE_FETCH_STATE_UNSPECIFIED\n"
]
}
],
"source": [
"url = \"https://talkingtochatbots.com/talking-to-chatbots/culture-and-entertainment/books-movies-and-tv/improving-alt-text-for-a-comic-strip-featuring-rick-sanchez-from-rick-and-morty/\"\n",
"\n",
"# Define the request body for URL Inspection\n",
"url_inspection_request = {\n",
" \"inspectionUrl\": url, \n",
" \"siteUrl\": 'sc-domain:talkingtochatbots.com', \n",
" \"languageCode\": \"en-US\"\n",
"}\n",
"\n",
"# Make the POST request\n",
"url_inspection_response = service.urlInspection().index().inspect(\n",
" body=url_inspection_request\n",
").execute()\n",
"\n",
"print(url_inspection_response)\n",
"\n",
"index_status_result = url_inspection_response.get('inspectionResult').get('indexStatusResult')\n",
"for key, value in index_status_result.items():\n",
" print(f\"{key}: {value}\")"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Discovered - currently not indexed'"
]
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"index_status_result.get('coverageState')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Inspect all pages in a directory"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['https://talkingtochatbots.com/bing/business-and-finance/', 'https://talkingtochatbots.com/bing/business-and-finance/are-internally-developed-software-included-in-capex/', 'https://talkingtochatbots.com/bing/business-and-finance/bernard-baruch-s-quote-meaning-his-1929-stock-positions-notable/', 'https://talkingtochatbots.com/bing/business-and-finance/comparison-of-stock-growth-for-emc-ntap-orcl-against-nasdaq-since/', 'https://talkingtochatbots.com/bing/business-and-finance/create-a-table-template-for-basic-financial-metrics-of-publicly/', 'https://talkingtochatbots.com/bing/business-and-finance/examples-of-companies-changing-dividend-policies-to-favor-stock/', 'https://talkingtochatbots.com/bing/business-and-finance/is-there-an-award-for-finance-investigative-journalism-or-notable/', 'https://talkingtochatbots.com/bing/business-and-finance/news-articles-about-recent-nft-mania-and-large-dollar-amounts-paid/', 'https://talkingtochatbots.com/bing/business-and-finance/recent-market-cap-price-metrics-and-yoy-growth-calculations-for/', 'https://talkingtochatbots.com/bing/business-and-finance/suggest-options-for-a-humorous-social-media-post-about-being-a/', 'https://talkingtochatbots.com/bing/business-and-finance/summarize-margin-call-and-find-a-related-insightful-stock-market-quote/', 'https://talkingtochatbots.com/bing/business-and-finance/summarize-the-plot-of-startup-com-and-provide-financial-metrics-for/', 'https://talkingtochatbots.com/bing/business-and-finance/discuss-trading-outlook-and-create-graphic-art-of-the-tesla-logo/', 'https://talkingtochatbots.com/bing/business-and-finance/what-are-some-terms-to-indicate-people-who-have-debt/', 'https://talkingtochatbots.com/bing/business-and-finance/what-are-the-most-famous-stock-market-indexes-for-technology/', 'https://talkingtochatbots.com/bing/business-and-finance/what-are-the-most-popular-stock-market-information-sites/', 'https://talkingtochatbots.com/bing/business-and-finance/what-is-the-60-40-portfolio-strategy/', 'https://talkingtochatbots.com/bing/business-and-finance/what-is-the-yen-carry-trade/', 'https://talkingtochatbots.com/bing/business-and-finance/what-percentage-of-adult-people-are-debt-free-in-the-united-states/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/animal-collective-songs-mentioning-animals-cats-dogs-and-indie-rock/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/create-an-informal-caption-for-mc-lars-outdated-song-download-this/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/describe-the-playstation-video-game-road-rash-in-a-couple-of-sentences/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/find-catchy-pop-or-rock-songs-related-to-dogs-woodwork-and-wood/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/find-kanye-west-lyrics-about-coming-back-excluding-the-song-homecoming/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/find-popular-songs-around-120-bpm-that-synchronize-with-a-pendulum/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/how-can-i-gain-followers-for-social-media-content-without-the/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/networking-lessons-from-mario-puzo-s-the-godfather-and-a-request-for/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/request-to-center-an-image-in-embed-code-for-mona-lisa-by-leonardo-da/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/what-indie-rock-indie-pop-or-electronic-songs-reference-contemplating/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/what-is-bluesky-and-its-purpose-as-a-website/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/what-is-the-average-rate-of-likes-to-views-on-tiktok/', 'https://talkingtochatbots.com/bing/language-and-writing/', 'https://talkingtochatbots.com/bing/language-and-writing/clarify-misconceptions-about-large-language-models-text-generation/', 'https://talkingtochatbots.com/bing/language-and-writing/comparative-review-of-grammarly-and-quillbot-including-differences/', 'https://talkingtochatbots.com/bing/language-and-writing/complete-the-phrase-from-roland-garros-court-la-victoire-appartient/', 'https://talkingtochatbots.com/bing/language-and-writing/correct-usage-of-whether-phrases-and-translation-options-for/', 'https://talkingtochatbots.com/bing/language-and-writing/create-sentences-using-specific-words-fine-however-unfortunately-not/', 'https://talkingtochatbots.com/bing/language-and-writing/creating-a-chatbot-prompt-for-humor-inspired-by-peter-sellers/', 'https://talkingtochatbots.com/bing/language-and-writing/derogatory-terms-for-people-who-excessively-enjoy-flattery-and-their/', 'https://talkingtochatbots.com/bing/language-and-writing/discussing-the-song-shout-by-tears-for-fears-and-potential-blog/', 'https://talkingtochatbots.com/bing/language-and-writing/events-and-news-in-the-eu-and-spain-related-to-tweets-from-october-13/', 'https://talkingtochatbots.com/bing/language-and-writing/find-a-quote-about-people-who-frequently-cite-quotes/', 'https://talkingtochatbots.com/bing/language-and-writing/find-funny-song-lyrics-verses-containing-the-word-professional/', 'https://talkingtochatbots.com/bing/language-and-writing/find-rock-or-pop-songs-with-pancake-in-lyrics-excluding-it-from-titles/', 'https://talkingtochatbots.com/bing/language-and-writing/fix-critical-writing-issues-in-the-text-and-recommend-songs-for-a-dog/', 'https://talkingtochatbots.com/bing/language-and-writing/how-do-you-name-individual-pieces-of-dog-food-in-english-similar-to/', 'https://talkingtochatbots.com/bing/language-and-writing/improve-social-media-post-and-comments-about-us-consumer-lending/', 'https://talkingtochatbots.com/bing/language-and-writing/improve-writing-on-coinbase-s-regulatory-challenges-and-sec-lawsuit/', 'https://talkingtochatbots.com/bing/language-and-writing/introduce-bear-in-a-message-about-twitter-and-uefa-champions-league/', 'https://talkingtochatbots.com/bing/language-and-writing/list-all-titles-of-queen-isabel-i-of-castile-separated-by-commas/', 'https://talkingtochatbots.com/bing/language-and-writing/promote-nvidia-s-story-using-humorous-sentences-about-gpus-gold-and/', 'https://talkingtochatbots.com/bing/language-and-writing/provide-better-phrase-alternatives-for-a-sentence-about-zwift-rollers/', 'https://talkingtochatbots.com/bing/language-and-writing/quotes-and-sayings-about-delivering-content-based-on-audience-demand/', 'https://talkingtochatbots.com/bing/language-and-writing/rephrase-taglines-create-humorous-polls-and-suggest-catchy-phrases/', 'https://talkingtochatbots.com/bing/language-and-writing/request-for-jokes-about-ceos-and-additional-information/', 'https://talkingtochatbots.com/bing/language-and-writing/request-to-identify-a-term-for-shorter-words-created-from-letters-of/', 'https://talkingtochatbots.com/bing/language-and-writing/suggest-names-and-explanations-for-a-stock-market-index-featuring/', 'https://talkingtochatbots.com/bing/language-and-writing/terms-for-errors-in-text-to-image-creations-like-poorly-drawn-hands/', 'https://talkingtochatbots.com/bing/language-and-writing/what-is-the-meaning-of-the-word-berlanguiano/', 'https://talkingtochatbots.com/bing/language-and-writing/what-s-the-most-appropriate-verb-to-complete-the-sentence-about-actor/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/analyze-the-question-are-we-all-f-ed-from-the-perspectives-of-five/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/complete-the-sentence-about-ai-and-emerging-technologies-role-in/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/do-you-know-a-philosopher-famous-for-his-frugal-lifestyle/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/find-cultural-references-or-quotes-for-a-social-media-message-about/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/find-indie-rock-songs-with-sunset-in-the-lyrics-but-not-in-the-title/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/find-quotes-song-lyrics-and-social-media-posts-about-sundays-and/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/find-upbeat-song-lyrics-referencing-drinking-from-fountains/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/find-upbeat-songs-with-appropriate-tempo-and-humorous-lyrics-for-a/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/how-does-mastodon-social-compare-to-other-mastodon-servers/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/how-often-are-uk-general-elections-held/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/identify-complex-scientific-topics-with-divided-theories-for-satire/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/is-there-a-stadium-with-credit-in-its-name/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/metals-in-the-list-lithium-nickel-sodium-cobalt-manganese-inquire/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/popular-twitter-hashtags-for-retweeting-old-tweets-humorously/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/request-to-read-a-web-page-inquire-about-a-linkedin-profile-and/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/shortlist-of-5-greatest-erudites-post-20th-century/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/should-i-use-theoretic-content-or-theoretical-content/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/social-media-reach-definition-and-alternative-suggestions-for/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-are-the-functionalities-and-workings-of-quora-spaces/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-can-savas-refer-to-in-the-israel-palestine-conflict-context/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-do-you-know-about-phil-sledge-on-twitter/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-does-squiggle-mean-according-to-michael-a-gayed-cfa-and-who-are/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-is-an-equivalent-term-for-day-to-day-conversations/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-is-the-difference-between-using-the-same-seed-and-a-start-image/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-is-the-opposite-of-close-up-in-photography/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-scientific-theories-including-lamarck-s-consider-human-limb/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/which-book-by-stanislaw-lem-should-i-read-next-after-solaris/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/who-is-david-gonz-lez-romero-and-what-are-his-skills-compared-to/', 'https://talkingtochatbots.com/bing/technology-and-ai/', 'https://talkingtochatbots.com/bing/technology-and-ai/ai-ethics-challenges-society-s-future-explore-definitions-of-art-ai/', 'https://talkingtochatbots.com/bing/technology-and-ai/alternatives-to-rephrase-sentences-about-aaa-bbb-quantum-computing/', 'https://talkingtochatbots.com/bing/technology-and-ai/are-microsoft-and-google-using-nvidia-chips-for-ai-and-who-has-a/', 'https://talkingtochatbots.com/bing/technology-and-ai/blogs-featuring-embedded-spotify-playlists-specifically-music-related/', 'https://talkingtochatbots.com/bing/technology-and-ai/cboe-volatility-index-description-us-treasury-bonds-2s-10s-yield/', 'https://talkingtochatbots.com/bing/technology-and-ai/compare-preferences-between-two-versions-of-a-statement-about-crh/', 'https://talkingtochatbots.com/bing/technology-and-ai/creative-fail-message-for-google-recaptcha-and-favorite-options/', 'https://talkingtochatbots.com/bing/technology-and-ai/discuss-ibm-s-quote-nobody-gets-fired-for-buying-ibm-its-historical/', 'https://talkingtochatbots.com/bing/technology-and-ai/discussing-conventional-wisdom-by-built-to-spill-lyrics-and-social/', 'https://talkingtochatbots.com/bing/technology-and-ai/equivalent-terms-for-websites-with-high-search-engine-authority/', 'https://talkingtochatbots.com/bing/technology-and-ai/examples-of-image-to-image-ai-models-and-websites-for-depixelating/', 'https://talkingtochatbots.com/bing/technology-and-ai/examples-of-text-based-generative-ai-models-including-llms/', 'https://talkingtochatbots.com/bing/technology-and-ai/explaining-stereotypes-in-movies-and-misconceptions-about-generative/', 'https://talkingtochatbots.com/bing/technology-and-ai/exploring-terms-related-to-compulsive-scrolling-and-rewriting/', 'https://talkingtochatbots.com/bing/technology-and-ai/how-does-the-geforce-rtx-2050-4gb-gpu-compare-to-other-graphics-cards/', 'https://talkingtochatbots.com/bing/technology-and-ai/htri-software-s-consideration-of-outer-diameter-correction-in-tube/', 'https://talkingtochatbots.com/bing/technology-and-ai/ideas-for-creating-a-custom-chatbot-and-advantages-over-bing-s/', 'https://talkingtochatbots.com/bing/technology-and-ai/identifying-terms-for-information-overload-and-improving-a-comment/', 'https://talkingtochatbots.com/bing/technology-and-ai/improve-the-style-of-texts-about-warehouse-automation-company-xyz-and/', 'https://talkingtochatbots.com/bing/technology-and-ai/improve-various-texts-and-clarify-ai-assistance-for-social-media/', 'https://talkingtochatbots.com/bing/technology-and-ai/improve-writing-about-chatbot-technology-recommending-prompt/', 'https://talkingtochatbots.com/bing/technology-and-ai/is-runway-ml-free-to-use/', 'https://talkingtochatbots.com/bing/technology-and-ai/is-the-paid-version-of-chatgpt-able-to-browse-the-internet-like-bing/', 'https://talkingtochatbots.com/bing/technology-and-ai/is-there-a-term-for-posting-the-same-text-on-multiple-social-media/', 'https://talkingtochatbots.com/bing/technology-and-ai/is-wordpress-a-fediverse-platform-and-how-does-pterotype-connect-to-it/', 'https://talkingtochatbots.com/bing/technology-and-ai/keywords-for-analyzing-search-terms-related-to-get-rich-quick-schemes/', 'https://talkingtochatbots.com/bing/technology-and-ai/list-of-ai-chatbots-including-free-options-and-hugging-face-s/', 'https://talkingtochatbots.com/bing/technology-and-ai/michael-a-gayed-s-articles-on-marketwatch-com-and-his-recent-views-on/', 'https://talkingtochatbots.com/bing/technology-and-ai/popular-ai-applications-search-terms-brands-and-websites-related-to/', 'https://talkingtochatbots.com/bing/technology-and-ai/popular-social-media-keywords-from-2009-2011-that-are-rarely-used/', 'https://talkingtochatbots.com/bing/technology-and-ai/recent-news-on-scammers-caught-using-caller-id-spoofing-in-the-last/', 'https://talkingtochatbots.com/bing/technology-and-ai/recommend-references-on-blockchain-and-crypto-assets-classifying-them/', 'https://talkingtochatbots.com/bing/technology-and-ai/request-for-spanish-term-expressing-technology-commoditization/', 'https://talkingtochatbots.com/bing/technology-and-ai/reword-and-fix-text-for-an-investment-platform-profile-about/', 'https://talkingtochatbots.com/bing/technology-and-ai/sales-pitch-for-a-cognitive-solution-assessing-customer-sentiment-in/', 'https://talkingtochatbots.com/bing/technology-and-ai/should-i-change-my-comment-on-schumpeter-s-quote-about-technology-and/', 'https://talkingtochatbots.com/bing/technology-and-ai/suggestions-for-a-cover-picture-to-enhance-engagement-on-an-ai/', 'https://talkingtochatbots.com/bing/technology-and-ai/suggestions-to-improve-text-quality-for-ai-powered-content-engagement/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-are-negative-and-positive-opinions-about-ai-art/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-are-online-communities-for-sharing-ai-chatbot-conversations/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-are-popular-company-profiles-on-mastodon-and-how-to-identify-my/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-are-the-countries-where-starlik-internet-service-is-currently/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-are-the-differences-similarities-and-syndication-benefits-of/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-are-the-most-popular-npc-hashtags-on-tiktok/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-are-the-typical-facebook-users-and-differences-between-facebook/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-consumer-applications-currently-use-llms-and-is-amazon-alexa/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-emojis-are-suitable-for-tagging-haute-cuisine-food/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-is-cuda/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-is-known-about-david-gonz-lez-romero-ict-engineer-and-business/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-is-ssl/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-is-the-difference-between-an-ai-model-and-an-ai-algorithm/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-is-the-most-popular-rss-reader/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-is-the-working-mechanism-of-the-iter-tokamak/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-mobile-communication-technology-replaced-gsm/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-were-the-most-popular-websites-and-internet-applications-in-the/', 'https://talkingtochatbots.com/bing/technology-and-ai/which-companies-are-the-leaders-in-cloud-storage/', 'https://talkingtochatbots.com/bing/technology-and-ai/who-are-the-current-most-followed-accounts-on-x-formerly-twitter/']\n",
"148\n"
]
}
],
"source": [
"url = 'https://talkingtochatbots.com/bing/'\n",
"\n",
"response = requests.get(url)\n",
"soup = BeautifulSoup(response.content, 'html.parser')\n",
"div = soup.find('div', class_='post-entry')\n",
"links = [a['href'] for a in div.find_all('a', href=True)]\n",
"ttcb_links = [link for link in links if link.startswith('https://talkingtochatbots.com')]\n",
"print(ttcb_links)\n",
"print(len(ttcb_links))"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['https://talkingtochatbots.com/bing/technology-and-ai/alternatives-to-rephrase-sentences-about-aaa-bbb-quantum-computing/', 'https://talkingtochatbots.com/bing/culture-and-entertainment/', 'https://talkingtochatbots.com/bing/technology-and-ai/recommend-references-on-blockchain-and-crypto-assets-classifying-them/', 'https://talkingtochatbots.com/bing/technology-and-ai/what-mobile-communication-technology-replaced-gsm/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/what-is-the-opposite-of-close-up-in-photography/', 'https://talkingtochatbots.com/bing/business-and-finance/what-are-the-most-popular-stock-market-information-sites/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/find-quotes-song-lyrics-and-social-media-posts-about-sundays-and/', 'https://talkingtochatbots.com/bing/science-and-general-knowledge/popular-twitter-hashtags-for-retweeting-old-tweets-humorously/', 'https://talkingtochatbots.com/bing/language-and-writing/provide-better-phrase-alternatives-for-a-sentence-about-zwift-rollers/', 'https://talkingtochatbots.com/bing/technology-and-ai/suggestions-to-improve-text-quality-for-ai-powered-content-engagement/']\n"
]
}
],
"source": [
"# Selecting a sample for demonstration purposes\n",
"ttcb_links = random.sample(ttcb_links, min(10, len(ttcb_links)))\n",
"print(ttcb_links)"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 10%|█ | 1/10 [00:06<00:58, 6.47s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/technology-and-ai/alternatives-to-rephrase-sentences-about-aaa-bbb-quantum-computing/: Discovered - currently not indexed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 20%|██ | 2/10 [00:12<00:51, 6.44s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/culture-and-entertainment/: URL is unknown to Google\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 30%|███ | 3/10 [00:19<00:44, 6.40s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/technology-and-ai/recommend-references-on-blockchain-and-crypto-assets-classifying-them/: Discovered - currently not indexed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 40%|████ | 4/10 [00:25<00:38, 6.39s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/technology-and-ai/what-mobile-communication-technology-replaced-gsm/: Discovered - currently not indexed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 50%|█████ | 5/10 [00:32<00:32, 6.45s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/science-and-general-knowledge/what-is-the-opposite-of-close-up-in-photography/: Discovered - currently not indexed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 60%|██████ | 6/10 [00:38<00:25, 6.41s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/business-and-finance/what-are-the-most-popular-stock-market-information-sites/: URL is unknown to Google\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 70%|███████ | 7/10 [00:44<00:19, 6.42s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/science-and-general-knowledge/find-quotes-song-lyrics-and-social-media-posts-about-sundays-and/: Discovered - currently not indexed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 80%|████████ | 8/10 [00:51<00:12, 6.43s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/science-and-general-knowledge/popular-twitter-hashtags-for-retweeting-old-tweets-humorously/: Discovered - currently not indexed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 90%|█████████ | 9/10 [00:57<00:06, 6.44s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/language-and-writing/provide-better-phrase-alternatives-for-a-sentence-about-zwift-rollers/: Discovered - currently not indexed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Inspecting URLs: 100%|██████████| 10/10 [01:04<00:00, 6.43s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://talkingtochatbots.com/bing/technology-and-ai/suggestions-to-improve-text-quality-for-ai-powered-content-engagement/: Submitted and indexed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"url_status = {}\n",
"for url in tqdm(ttcb_links, desc=\"Inspecting URLs\"):\n",
" # Define the request body for URL Inspection\n",
" url_inspection_request = {\n",
" \"inspectionUrl\": url, \n",
" \"siteUrl\": 'sc-domain:talkingtochatbots.com', \n",
" \"languageCode\": \"en-US\"\n",
" }\n",
"\n",
" # Make the POST request\n",
" url_inspection_response = service.urlInspection().index().inspect(\n",
" body=url_inspection_request\n",
" ).execute()\n",
"\n",
" index_status_result = url_inspection_response.get('inspectionResult').get('indexStatusResult').get('coverageState')\n",
" url_status[url] = index_status_result\n",
" print(f\"{url}: {index_status_result}\")"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>URL</th>\n",
" <th>Status</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>https://talkingtochatbots.com/bing/technology-...</td>\n",
" <td>Discovered - currently not indexed</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>https://talkingtochatbots.com/bing/culture-and...</td>\n",
" <td>URL is unknown to Google</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>https://talkingtochatbots.com/bing/technology-...</td>\n",
" <td>Discovered - currently not indexed</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>https://talkingtochatbots.com/bing/technology-...</td>\n",
" <td>Discovered - currently not indexed</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>https://talkingtochatbots.com/bing/science-and...</td>\n",
" <td>Discovered - currently not indexed</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>https://talkingtochatbots.com/bing/business-an...</td>\n",
" <td>URL is unknown to Google</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>https://talkingtochatbots.com/bing/science-and...</td>\n",
" <td>Discovered - currently not indexed</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>https://talkingtochatbots.com/bing/science-and...</td>\n",
" <td>Discovered - currently not indexed</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>https://talkingtochatbots.com/bing/language-an...</td>\n",
" <td>Discovered - currently not indexed</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>https://talkingtochatbots.com/bing/technology-...</td>\n",
" <td>Submitted and indexed</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" URL \\\n",
"0 https://talkingtochatbots.com/bing/technology-... \n",
"1 https://talkingtochatbots.com/bing/culture-and... \n",
"2 https://talkingtochatbots.com/bing/technology-... \n",
"3 https://talkingtochatbots.com/bing/technology-... \n",
"4 https://talkingtochatbots.com/bing/science-and... \n",
"5 https://talkingtochatbots.com/bing/business-an... \n",
"6 https://talkingtochatbots.com/bing/science-and... \n",
"7 https://talkingtochatbots.com/bing/science-and... \n",
"8 https://talkingtochatbots.com/bing/language-an... \n",
"9 https://talkingtochatbots.com/bing/technology-... \n",
"\n",
" Status \n",
"0 Discovered - currently not indexed \n",
"1 URL is unknown to Google \n",
"2 Discovered - currently not indexed \n",
"3 Discovered - currently not indexed \n",
"4 Discovered - currently not indexed \n",
"5 URL is unknown to Google \n",
"6 Discovered - currently not indexed \n",
"7 Discovered - currently not indexed \n",
"8 Discovered - currently not indexed \n",
"9 Submitted and indexed "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"url_status_df = pd.DataFrame(list(url_status.items()), columns=['URL', 'Status'])\n",
"display(url_status_df)"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [],
"source": [
"url_status_df.to_csv('url_status.csv', index=False)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "base",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment