4 replies to this topic

[Adam]

I've never been able to fully grasp how to parse [GET] pet names or codestones from the neopets website and have it [POST] to my auto trainer. I need to post the pet name to a text box. I've never been able to finish my projects that have to do with parsing. This is what my Auto Trainer looks like so far, mind you I've only started working on it about an hour ago. I have the login working..lol

[attachment=10235:halp.png]

If someone such as Waser wants my project, I'd be happy to send it to ya.

Here is the wrapper I'm using.

Spoiler

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using System.Text.RegularExpressions;
using System.IO;
using System.IO.Compression;
using System.Net.Sockets;

namespace goodwrap
{
	class wrapper
	{
		private TcpClient client;
		IDictionary<string, string> colCookies = new Dictionary<string, string>();
		public string strCookies;
		public string LastPage;

		public string Request(string Method, string URL, string Referer)
		{
			string Host = null;
			string strFile = null;
			string strPost = null;
			int pos = 0;

			if (Referer == null)
			{
				Referer = LastPage;
			}
			if (URL.Contains("http://"))
			{
				Host = URL.Substring(7);
			}
			else
			{
				Host = URL;
			}
			if (Host.Contains("/"))
			{
				pos = Host.IndexOf("/", 0);
				strFile = Host.Substring(pos);
				Host = Host.Substring(0, pos);
			}
			else
			{
				strFile = "/";
			}
			if (Method == "POST")
			{
				pos = strFile.IndexOf("?");
				if (pos != -1)
				{
					strPost = strFile.Substring(pos + 1);
					strFile = strFile.Substring(0, pos);
				}
				else
				{
					strPost = null;
				}
			}
			LastPage = URL;

			string ReqHeaders = null;
			if (Method == "GET" || Method == "PIC")
			{
				ReqHeaders = "GET" + " " + strFile + " HTTP/1.1" + "\r\n"
				+ "Host: " + Host + "\r\n"
				+ "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.4) Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)" + "\r\n"
				+ "Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/ *;q=0.5" + "\r\n"
				+ "Accept-Language: en-us,en;q=0.5" + "\r\n"
				+ "Accept-Encoding: gzip, deflate" + "\r\n"
				+ "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7" + "\r\n"
				+ "Keep-Alive: 300" + "\r\n"
				+ "Connection: keep-alive" + "\r\n"
				+ "Referer: " + Referer + "\r\n"
				+ "Cookie: " + strCookies + "\r\n" + "\r\n";
			}
			else
			{
				ReqHeaders = "POST " + strFile + " HTTP/1.1" + "\r\n"
				+ "Host: " + Host + "\r\n"
				+ "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.4) Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)" + "\r\n"
				+ "Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/ *;q=0.5" + "\r\n"
				+ "Accept-Language: en-us,en;q=0.5" + "\r\n"
				+ "Accept-Encoding: gzip, deflate" + "\r\n"
				+ "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7" + "\r\n"
				+ "Keep-Alive: 300" + "\r\n"
				+ "Connection: keep-alive" + "\r\n"
				+ "Referer: " + Referer + "\r\n"
				+ "Cookie: " + strCookies + "\r\n"
				+ "Content-Type: application/x-www-form-urlencoded" + "\r\n"
				+ "Content-Length: " + strPost.Length.ToString() + "\r\n"
				+ "Connection: close" + "\r\n" + "\r\n"
				+ strPost;
			}
			if (Method == "PIC") ReqHeaders.Replace("Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/ *;q=0.5", "Accept: image/png,*/*;q=0.5");

			client = new TcpClient(Host, 80);
			Byte[] headers = System.Text.Encoding.ASCII.GetBytes(ReqHeaders);
			NetworkStream ns = client.GetStream();
			ns.Write(headers, 0, headers.Length);
			StreamReader sr = new StreamReader(ns, Encoding.Default);
			String strHTML = sr.ReadToEnd();

			string[] strParts = Regex.Split(strHTML, Environment.NewLine + Environment.NewLine);
			strCookies = ParseCookies(strParts[0]);
			if (strParts[0].Contains("Content-Encoding"))
			{
				strParts[1] = DecompressGzip(strParts[1]);
			}
			return strParts[0] + Environment.NewLine + Environment.NewLine + strParts[1];
		}

		public string DecompressGzip(string compressed)
		{
			MemoryStream memStream = new MemoryStream(System.Text.Encoding.Default.GetBytes(compressed));
			GZipStream decompressStream = new GZipStream(memStream, CompressionMode.Decompress);

			byte[] endBytes = new byte[4];
			int position = (int)memStream.Length - 4;
			memStream.Position = position;
			memStream.Read(endBytes, 0, 4);
			memStream.Position = 0;
			byte[] buffer = new byte[BitConverter.ToInt32(endBytes, 0) + 100];
			int offset = 0;
			int total = 0;
			while (true)
			{
				int o = decompressStream.Read(buffer, offset, 100);
				if (o == 0) break;
				offset += o;
				total += o;
			}
			return Encoding.ASCII.GetString(buffer);
		}

		public string NeoLogin(string user, string pass, ref bool loggedIn)
		{
			string strHTML = null;
			Request("GET", "http://neopets.com/loginpage.phtml", "http://google.com");
			Pause(1);
			Request("POST", "http://www.neopets.com/hi.phtml?destination=%2Fpetcentral.phtml&username=" + user, "http://neopets.com/loginpage.phtml");
			Pause(1);
			strHTML = Request("POST", "http://www.neopets.com/login.phtml?username=" + user + "&password=" + pass + "&destination=%2Fpetcentral.phtml", "http://neopets.com/hi.phtml");
			if (strHTML.Contains("Set-Cookie: neologin="))
			{
				loggedIn = true;
				return "Logged In";
			}
			else if (strHTML.Contains("too many times"))
			{
				loggedIn = false;
				return "To Many Login Attempts";
			}
			else if (strHTML.Contains("badpassword"))
			{
				loggedIn = false;
				return "Wrong Password";
			}
			else if (strHTML.Contains("frozen"))
			{
				loggedIn = false;
				return "Account Frozen";
			}
			else if (strHTML.Contains("just a technical problem"))
			{
				loggedIn = false;
				return "Neopets is down for maintenance.";
			}
			else
			{
				loggedIn = false;
				return strHTML;
			}
		}

		private static void Pause(double seconds)
		{
			double num = seconds * 1000;
			DateTime t1 = DateTime.Now, t2 = DateTime.Now;
			TimeSpan tmDiff = t2 - t1;
			while (Convert.ToDouble(tmDiff.TotalMilliseconds.ToString()) < num)
			{
				t2 = DateTime.Now;
				tmDiff = t2 - t1;
				Application.DoEvents();
			}
		}

		public string StripHeaders(string strSource)
		{
			string[] strParts = Regex.Split(strSource, Environment.NewLine + Environment.NewLine);
			return strParts[1];
		}

		public Bitmap GrabPic(string strURL)
		{
			MemoryStream memStream = new MemoryStream(System.Text.Encoding.Default.GetBytes(StripHeaders(Request(" GET", strURL, LastPage))));
			Bitmap bitmap = new Bitmap(memStream);
			return bitmap;
		}

		public void ClearCookies()
		{
			colCookies.Clear();
			strCookies = null;
		}

		public string ParseCookies(string Headers)
		{//Credit's to Mystical for RegEx
			string ParseCookies = null;
			MatchCollection matches;
			Regex reg = new Regex(@"Set-Cookie:\s*(.+?)=(.+?);", RegexOptions.IgnoreCase);
			if (reg.IsMatch(Headers))
			{
				matches = reg.Matches(Headers);
				foreach (Match m in matches)
				{
					if (colCookies.ContainsKey(m.Groups[1].ToString()))
					{
						colCookies.Remove(m.Groups[1].ToString());
						colCookies.Add(m.Groups[1].ToString(), m.Groups[1].ToString() + "=" + m.Groups[2].ToString());
					}
					else
					{
						colCookies.Add(m.Groups[1].ToString(), m.Groups[1].ToString() + "=" + m.Groups[2].ToString());
					}
				}
			}
			foreach (KeyValuePair<string, string> item in colCookies)
			{
				ParseCookies = ParseCookies + item.Value.ToString() + "; ";
			}
			return ParseCookies;

		}
	}
}

Edited by Adam, 15 May 2012 - 04:27 PM.

[Waser Lave]

You can either use something like regular expressions to parse the data using patterns or you can use basic string manipulation like String.SubString().

[artificial]

Basically the idea is you use GET to request the source code of that page, you use various string parsing techniques to retrieve the pet name(s) within that page source, and use that information when you POST to whatever page you want.

You may want to look in to some regex tutorials for .NET (the good thing about regex is syntax wise the patterns are almost universal across all languages, but implementation can change language to language). Here's a brief tutorial I found on Google that may help you: http://www.codeproje...-Regex-Tutorial

[Adam]

Okay I think I've gotten it..but I've ran into a small problem...
I changed the layout of my program to something very different, but when I compile it the GUI goes back to how I originally had it. Anyone care to explain why?

Oh also:

var strHTML = myWrapper.Request("GET", "http://www.neopets.com/island/process_training.phtml?type=pay&pet_name=MYPETNAME"
+ itemList.Items[i].ToString() + "http://www.neopets.com/island/process_training.phtml?type=start&course_type=COURSE", "http://www.neopets.com/island/training.phtml?type=courses");
				// Pause(1);
				courseText = GrabAll(strHTML, "option value='S", " H");

courseText = GrabAll(strHTML, "option value='S", " H"); is supposed to grab the course value and translate it to be Strength...not sure if it's right.

Edited by Adam, 15 May 2012 - 08:50 PM.

[data]

Okay I think I've gotten it..but I've ran into a small problem...
I changed the layout of my program to something very different, but when I compile it the GUI goes back to how I originally had it. Anyone care to explain why?

Oh also:

var strHTML = myWrapper.Request("GET", "http://www.neopets.com/island/process_training.phtml?type=pay&pet_name=MYPETNAME"
+ itemList.Items[i].ToString() + "http://www.neopets.com/island/process_training.phtml?type=start&course_type=COURSE", "http://www.neopets.com/island/training.phtml?type=courses");
				// Pause(1);
				courseText = GrabAll(strHTML, "option value='S", " H");

courseText = GrabAll(strHTML, "option value='S", " H"); is supposed to grab the course value and translate it to be Strength...not sure if it's right.

Post your 'GrabAll' function. What do you mean the GUI resets? Do all the buttons get removed and does it return to a blank moveable form?