trip

Author: mercuryminds-scrapy

c.py

@@ -0,0 +1 @@
+__author__ = 'anandhakumar'

d/urbanspoon/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# http://doc.scrapy.org/topics/scrapyd.html
+
+[settings]
+default = urbanspoon.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = urbanspoon

d/urbanspoon/urbanspoon/__init__.py

@@ -0,0 +1,22 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# http://doc.scrapy.org/en/latest/topics/items.html
+
+from scrapy.item import Item, Field
+
+class UrbanspoonItem(Item):
+    # define the fields for your item here like:
+    # name = Field()
+    b_id = Field()
+    url = Field()
+    n_votes = Field()
+    p_like= Field()
+    n_reviews= Field()
+    date= Field()
+    title= Field()
+    description= Field()
+    user = Field()
+
+
+

d/urbanspoon/urbanspoon/items.py

@@ -0,0 +1,8 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: http://doc.scrapy.org/topics/item-pipeline.html
+
+class UrbanspoonPipeline(object):
+    def process_item(self, item, spider):
+        return item

d/urbanspoon/urbanspoon/pipelines.py

@@ -0,0 +1,15 @@
+# Scrapy settings for Urbanspoon project
+#
+# For simplicity, this file contains only the most important settings by
+# default. All the other settings are documented here:
+#
+#     http://doc.scrapy.org/topics/settings.html
+#
+
+BOT_NAME = 'urbanspoon'
+
+SPIDER_MODULES = ['urbanspoon.spiders']
+NEWSPIDER_MODULE = 'urbanspoon.spiders'
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'urbanspoon (+http://www.yourdomain.com)'

d/urbanspoon/urbanspoon/settings.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

d/urbanspoon/urbanspoon/spiders/__init__.py

@@ -0,0 +1,108 @@
+""" Enter Url here"""
+#url =['http://www.urbanspoon.com/r/35/1571097/restaurant/Hampton-Roads/Dam-Neck-Corner-Pungo/Firebrew-Virginia-Beach']
+url =['http://www.urbanspoon.com/r/1/4524/restaurant/Capitol-Hill/Honey-Hole-Sandwiches-Seattle','http://www.urbanspoon.com/r/35/1571097/restaurant/Hampton-Roads/Dam-Neck-Corner-Pungo/Firebrew-Virginia-Beach','http://www.urbanspoon.com/r/13/169913/restaurant/North-Richland-Hills-Richland-Hills/Texs-Star-Grill-Watauga']
+from scrapy.spider import BaseSpider
+from urlparse import urljoin
+from scrapy.http import Request
+from scrapy.selector import HtmlXPathSelector
+from urbanspoon.items import UrbanspoonItem
+import psycopg2
+
+
+""" Database Creation"""
+try:
+	con = psycopg2.connect(database="mm", user="postgres", password="mercuryminds", host="localhost")
+	cur = con.cursor()
+except:
+	""" Database not Connected """
+
+class UrbanspoonSpider(BaseSpider):
+	
+	name = "urban"
+
+#	allowed_domains = ["urbanspoon.com"]
+	start_urls=[url[i] for i in range(len(url))]
+
+#	filehandle = open('Urbanspoon.csv','w')
+#	filehandle.write("Manufacturer\tDescription\tPart Number\tUrl\tAvailability\tPrice\tItem Number\tOEM\tMore_items\n")
+	
+	def parse(self,response):
+
+		item=UrbanspoonItem()
+		hxs = HtmlXPathSelector(response)
+		item['url']= response.url
+		item['b_id'] = 0
+
+		"""Getting the Business Id from the DB if exists """
+
+		qu = ("select distinct url from social_data.urbanspoon")
+		cur.execute(qu)
+		rows = cur.fetchall()
+		li = [r[0] for r in rows]
+#		print li
+
+		for i in range(len(li)):
+	#		print li[i], item['url']
+			if li[i] in item['url']:
+				qu1 = ("select distinct business_id from social_data.urbanspoon where url='%s'")%li[i]
+				cur.execute(qu1)
+				rows1= cur.fetchall()
+				li1=[i[0] for i in rows1]
+#				print li1				
+				if len(li1) > 0:
+					item['b_id'] = li1[0]
+		if item['b_id'] > 0:
+			print " ID assigned"
+		else:
+			item['b_id'] = input("Enter the Business Id here for URL:       %s        : "%item['url'])
+
+
+		print item['b_id']
+
+
+		
+		item['n_votes'] = hxs.select('//div[@class="stats"]/div/text()').extract()[0].encode('utf-8').replace('\n','').strip()
+		item['p_like']	= hxs.select('//div[@class="rating"]/text()').extract()[0].encode('utf-8').strip()
+		item['n_reviews'] = int(hxs.select('//div[@class="stats"]/div/a[@data-ga-action="reviews"]/text()').extract()[0].split(' ')[0].encode('utf-8').strip())
+		nxt_link = 'http://www.urbanspoon.com'+hxs.select('//div[@data-ga-action="diner-reviews"]/@data-url').extract()[0].encode('utf-8').strip()
+		print nxt_link
+		if nxt_link:
+			yield Request(nxt_link, callback=self.parse_sub, meta=dict(item=item))
+	
+	def parse_sub(self,response):
+
+
+		print "Sub Parse Called"
+		item = response.meta.get('item')
+		hxs = HtmlXPathSelector(response)
+
+		x=hxs.select('//ul/li[@class="comment review"]')
+		length=len(x[0].select('//div[@class="details"]/div[@class="byline"]/a[@itemprop="reviewer"]/text()').extract())
+
+		for i in range(length):
+					
+			try:
+				item['date']  = x[0].select('//div[@class="details"]/div/time[@class="posted-on"]/text()').extract()[i].encode('utf-8').split(' ')[2].replace('\n','')
+				
+			except:
+				item['date'] = ''
+			try:
+				item['title'] = x[0].select('//div[@class="details"]/div[@class="title"]/text()').extract()[i].encode('utf-8').replace("'","").strip()
+			except:
+				item['title'] = ''
+				
+			try:
+				item['description'] = x[0].select('//div[@class="details"]/div[@itemprop="description"]').extract()[i].encode('utf-8').split('\n')[1].replace("'","")
+			except:
+				item['description'] = ''
+			try:
+				item['user'] = x[0].select('//div[@class="details"]/div[@class="byline"]/a[@itemprop="reviewer"]/text()').extract()[i].encode('utf-8').replace("'","").strip()
+			except:
+				item['user'] = ''
+			print item['user']
+			sql = ("insert into social_data.urbanspoon select '%s','%s','%s','%s','%s','%s','%s','%s','%s' where not exists ( select * from social_data.urbanspoon where review_description='%s' and review_user='%s')"%(item['b_id'],item['url'],item['n_votes'], item['p_like'], item['n_reviews'],item['date'],item['title'],item['description'], item['user'],item['description'], item['user']))
+		        cur.execute(sql)
+		        con.commit()
+
+		
+

d/urbanspoon/urbanspoon/spiders/urban.py

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# http://doc.scrapy.org/topics/scrapyd.html
+
+[settings]
+default = yelp.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = yelp