Python Web Stuff
#1
Posted 20 September 2010 - 08:16 PM
#2
Posted 20 September 2010 - 09:13 PM
#3
Posted 21 September 2010 - 02:17 AM
class httpClient: def __init__( self, UserAgent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.9) Gecko/20100824 Firefox/3.6.9' ): self.UserAgent = UserAgent; def Request(self, Type, Url, Referer = '', Data = None): Req = urllib2.Request( Url ) Req.add_header( 'Referer', Referer ) Req.add_header( 'User-Agent', self.UserAgent ) R = urllib2.urlopen( Req ) page = R.read() return page.encode('utf-8')
#4
Posted 21 September 2010 - 07:00 AM
It's really simple in python. With the time and energy it would take PMing those guys and grovelling your way in to Jr.Programmer you could just make one. I'll start you off:
class httpClient: def __init__( self, UserAgent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.9) Gecko/20100824 Firefox/3.6.9' ): self.UserAgent = UserAgent; def Request(self, Type, Url, Referer = '', Data = None): Req = urllib2.Request( Url ) Req.add_header( 'Referer', Referer ) Req.add_header( 'User-Agent', self.UserAgent ) R = urllib2.urlopen( Req ) page = R.read() return page.encode('utf-8')
That's so much simpler in python than in C variants. On a side note, sounds like you have experience asking to be a jr programmer.
Edited by Faval, 21 September 2010 - 07:01 AM.
#5
Posted 21 September 2010 - 07:07 AM
That's so much simpler in python than in C variants. On a side note, sounds like you have experience asking to be a jr programmer.
He used to be a Full Programmer here so yeah he has some experience.
#6
Posted 06 October 2010 - 12:58 PM
import cookielib, urllib2, time, os, re from datetime import datetime, timedelta class Url(object): def __init__(self): self.lastUrl = "" self.cookiejar = cookielib.CookieJar() self.requestOpener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookiejar)) self.rEvents = [] self.lastData = "" def open(self, url, cnt=0): try: req = self.request(url) reqO = self.requestOpener.open(req) resp = ResponseProxy(reqO) self.lastData = resp.read() resp.reset() time.sleep(0.1) return resp except urllib2.HTTPError, (err): if cnt < 10: time.sleep(3) return self.open(url, cnt+1) else: print "\n"+str(err)+"\n" return False def request(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False): req_headers = { 'User-Agent': '<GO GET A CLIENT TAG. GOOGLE IS THE BEST>', 'Keep-Alive': '300', 'Connection': 'keep-alive' } headers.update(req_headers) if self.lastUrl != "": headers.update({'Referer': self.lastUrl}) try: ret = urllib2.Request(url, data, headers, origin_req_host, unverifiable) self.lastUrl = url return ret except AttributeError: return url def setRefer(self, url=""): self.lastUrl = url def getRefer(self): return self.lastUrl class ResponseProxy(object): def __init__(self, response): self.resp = response self.data = self.resp.read() self.index = 0 def cliffHangerFix(self): self.data = self.data.replace("<input type='submit' value='Play Again'>", "<input type='submit' value='Play Again'></form>") self.data = self.data.replace("<input type='submit' value='Start Game'>", "<input type='submit' value='Start Game'></form>") def reset(self): self.index = 0 def read(self, n = None): if n == None: to_return = self.data[self.index:] self.index = len(self.data) return to_return to_return = self.data[self.index:self.index + n] self.index += n return to_return def __getattr__(self, name): if name in self.__dict__: return self.__dict__['name'] else: return getattr(self.resp, name) def GetBetween(zStr, zStart, zEnd): z1 = zStr.find(zStart) z2 = zStr.find(zEnd, z1+len(zStart)); if(z2 > z1 and z1 > -1): return zStr[z1+len(zStart):z2] else: return "";
basically use it like this
url = Url() response = url.open("http://www.neopets.com") html = response.read()
There is a bunch of other junk in there that i use for other parts that are not as important; step 1: getting it to work
#7
Posted 06 October 2010 - 01:35 PM
#8
Posted 06 October 2010 - 01:46 PM
Something else that might help is to look into a python library called "ClientForm" works great as a user-friendly module to send post data (such as logging into websites and submitting forms) Most of my URL module was designed to wrap around that so that i was able to create many simple to create scripts
http://wwwsearch.sou...old/ClientForm/
#9
Posted 06 October 2010 - 01:48 PM
#10
Posted 06 October 2010 - 02:00 PM
approximately 10,000 lines of python
over 50 different tasks
program runs on 2 ports
multi-threading
custom config file support
24/7 support
it all started as a way to learn a language (lesson to you noobies, just because a task seems hard, don't give up. check out this post, in the end i have a 53% win ratio in minesweeper; the best program i could find was only 32%) http://ubuntuforums....d.php?t=1286392
I guess I've gotten to the point where i want to experience team programming It gets a little lonely at night XD
Feel free to ask me any questions you guys have about programing in python Id love to contribute any way i can xD
~Cody
#11
Posted 06 October 2010 - 02:12 PM
0 user(s) are reading this topic
0 members, 0 guests, 0 anonymous users