-
-
Save timlinquist/1574070 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| require 'eventmachine' | |
| require 'em-http' | |
| c = 4 | |
| urls = DATA.readlines.map {|s| s.strip } | |
| pending = urls.size | |
| before = Time.now | |
| puts "urls: #{pending}" | |
| puts "concurrency: #{c}" | |
| EM.run do | |
| q = EM::Queue.new | |
| q.push(*urls) | |
| worker = lambda do |url| | |
| http = EM::HttpRequest.new("http://#{url}/").get(:timeout => 10) | |
| http.callback do | |
| pending -= 1 | |
| EM.stop if pending < 1 | |
| q.pop worker | |
| end | |
| http.errback do | |
| puts "#{url} #{http.error}" | |
| end | |
| http.errback do | |
| pending -= 1 | |
| EM.stop if pending < 1 | |
| q.pop worker | |
| end | |
| end | |
| c.times do | |
| q.pop worker | |
| end | |
| EM.add_periodic_timer(2) do | |
| puts "#{q.size}" | |
| end | |
| end | |
| puts "finished in #{Time.now - before}" | |
| # The first 300 from Alexa's top 500 domains: | |
| # http://www.alexa.com/topsites | |
| __END__ | |
| ebay.com | |
| bing.com | |
| google.fr | |
| microsoft.com | |
| yandex.ru | |
| 163.com | |
| google.co.jp | |
| linkedin.com | |
| myspace.com | |
| google.com.br | |
| craigslist.org | |
| conduit.com | |
| flickr.com | |
| fc2.com | |
| mail.ru | |
| google.it | |
| imdb.com | |
| vkontakte.ru | |
| livejasmin.com | |
| google.es | |
| googleusercontent.com | |
| mozilla.com | |
| rapidshare.com | |
| soso.com | |
| go.com | |
| sohu.com | |
| bbc.co.uk | |
| apple.com | |
| aol.com | |
| youku.com | |
| doubleclick.com | |
| xvideos.com | |
| google.com.mx | |
| bp.blogspot.com | |
| ask.com | |
| pornhub.com | |
| adobe.com | |
| google.ca | |
| google.ru | |
| paypal.com | |
| cnn.com | |
| orkut.com.br | |
| mediafire.com | |
| tudou.com | |
| youporn.com | |
| sogou.com | |
| espn.go.com | |
| photobucket.com | |
| hotfile.com | |
| xhamster.com | |
| 4shared.com | |
| about.com | |
| orkut.com | |
| megaupload.com | |
| ameblo.jp | |
| livejournal.com | |
| imageshack.us | |
| google.co.id | |
| hao123.com | |
| cnet.com | |
| rakuten.co.jp | |
| ebay.de | |
| google.com.tr | |
| megavideo.com | |
| google.com.au | |
| google.cn | |
| uol.com.br | |
| files.wordpress.com | |
| livedoor.com | |
| orkut.co.in | |
| thepiratebay.org | |
| google.pl | |
| nytimes.com | |
| alibaba.com | |
| ifeng.com | |
| tube8.com | |
| tianya.cn | |
| ebay.co.uk | |
| google.com.sa | |
| globo.com | |
| mixi.jp | |
| godaddy.com | |
| twitpic.com | |
| renren.com | |
| cpxinteractive.com | |
| hi5.com | |
| digg.com | |
| dailymotion.com | |
| odnoklassniki.ru | |
| kaixin001.com | |
| optmd.com | |
| yieldmanager.com | |
| redtube.com | |
| google.nl | |
| xtendmedia.com | |
| deviantart.com | |
| amazon.de | |
| ku6.com | |
| taringa.net | |
| netflix.com | |
| goo.ne.jp | |
| weather.com | |
| cnzz.com | |
| ezinearticles.com | |
| xnxx.com | |
| mywebsearch.com | |
| ehow.com | |
| spiegel.de | |
| rediff.com | |
| tumblr.com | |
| amazon.co.jp | |
| filestube.com | |
| google.com.ar | |
| aweber.com | |
| zedo.com | |
| xunlei.com | |
| google.co.th | |
| clicksor.com | |
| google.com.eg | |
| bankofamerica.com | |
| answers.com | |
| rambler.ru | |
| sourceforge.net | |
| ning.com | |
| google.com.pk | |
| google.co.za | |
| huffingtonpost.com | |
| nicovideo.jp | |
| youjizz.com | |
| chinaz.com | |
| wordpress.org | |
| fifa.com | |
| bit.ly | |
| badoo.com | |
| secureserver.net | |
| wretch.cc | |
| partypoker.com | |
| download.com | |
| addthis.com | |
| amazon.co.uk | |
| depositfiles.com | |
| adultfriendfinder.com | |
| terra.com.br | |
| maktoob.com | |
| bild.de | |
| torrentz.com | |
| indiatimes.com | |
| vk.com | |
| onet.pl | |
| chase.com | |
| 118114.cn | |
| angege.com | |
| 56.com | |
| free.fr | |
| tinypic.com | |
| vimeo.com | |
| gmx.net | |
| digitalpoint.com | |
| wikimedia.org | |
| nk.pl | |
| blogfa.com | |
| comcast.net | |
| stumbleupon.com | |
| douban.com | |
| alipay.com | |
| xinhuanet.com | |
| orange.fr | |
| statcounter.com | |
| typepad.com | |
| tagged.com | |
| dell.com | |
| xvideoslive.com | |
| fastclick.com | |
| dailymail.co.uk | |
| mybrowserbar.com | |
| gougou.com | |
| archive.org | |
| ameba.jp | |
| web.de | |
| hubpages.com | |
| google.be | |
| nifty.com | |
| domaintools.com | |
| hp.com | |
| 2ch.net | |
| people.com.cn | |
| google.co.ve | |
| reference.com | |
| metacafe.com | |
| wikia.com | |
| libero.it | |
| naver.com | |
| foxnews.com | |
| hulu.com | |
| pconline.com.cn | |
| wp.pl | |
| allegro.pl | |
| zynga.com | |
| xing.com | |
| guardian.co.uk | |
| zol.com.cn | |
| 4399.com | |
| skype.com | |
| mop.com | |
| marca.com | |
| google.com.vn | |
| google.gr | |
| imagevenue.com | |
| google.com.co | |
| squidoo.com | |
| soufun.com | |
| fastbrowsersearch.com | |
| isohunt.com | |
| google.at | |
| netlog.com | |
| clickbank.com | |
| walmart.com | |
| istockphoto.com | |
| amazonaws.com | |
| fbcdn.net | |
| biglobe.ne.jp | |
| liveinternet.ru | |
| opendns.com | |
| google.com.my | |
| telegraph.co.uk | |
| youdao.com | |
| google.se | |
| wsj.com | |
| gamespot.com | |
| spankwire.com | |
| linkbucks.com | |
| soku.com | |
| leboncoin.fr | |
| mozilla.org | |
| 126.com | |
| scribd.com | |
| 51.la | |
| ign.com | |
| hatena.ne.jp | |
| yelp.com | |
| kaskus.us | |
| tripadvisor.com | |
| google.ch | |
| google.ro | |
| vnexpress.net | |
| adultadworld.com | |
| ig.com.br | |
| uimserv.net | |
| ucoz.ru | |
| megaporn.com | |
| tweetphoto.com | |
| 58.com | |
| keezmovies.com | |
| tom.com | |
| reddit.com | |
| joomla.org | |
| leo.org | |
| cricinfo.com | |
| z5x.net | |
| google.com.ua | |
| booking.com | |
| rutracker.org | |
| mashable.com | |
| seznam.cz | |
| 360buy.com | |
| infoseek.co.jp | |
| narod.ru | |
| hardsextube.com | |
| google.com.pe | |
| cj.com | |
| mlb.com | |
| 51.com | |
| match.com | |
| mapquest.com | |
| brothersoft.com | |
| usps.com | |
| google.pt | |
| pornhublive.com | |
| csdn.net | |
| babylon.com | |
| 2345.com | |
| sakura.ne.jp | |
| avg.com | |
| geocities.jp | |
| softonic.com | |
| w3schools.com | |
| icio.us | |
| yesky.com | |
| google.cl | |
| ups.com | |
| repubblica.it | |
| salesforce.com | |
| admagnet.net | |
| google.com.ph | |
| love21cn.com | |
| reuters.com | |
| engadget.com | |
| expedia.com | |
| att.com | |
| fileserve.com |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment