import os, re, sys, urllib, urlparse
from stat import *

mainurl = "http://www.free-hardcore.com/menu.php"
ignore = ["meninpain"]

pattern1 = r'(?s)<table.*?>(.*?)</table>'
pattern2 = r'(?s)<tr.*?>(.*?)</tr>'
pattern3 = r'(?s)<td.*?>(.*?)</td>'
pattern4 = r'(?s)<a href="(.*?)".*?>(.*?)</a>'
pattern5 = r'(?s)<a href="(.*?)".*?>'

link = urllib.urlopen(mainurl)
page = link.read()
print mainurl

tables = re.findall(pattern1, page)
print len(tables), "tables"

rows = re.findall(pattern2, tables[0])
rows = rows[1:]
print len(rows), "rows"

for row in reversed(rows):
    cols = re.findall(pattern3, row)
    for col in cols:
        hrefs = re.findall(pattern4, col)
        for href, mode in hrefs:
            if mode != "video": continue
            parsed = urlparse.urlparse(href)
            domain = parsed[1].split(".")
            path = parsed[2].split("/")
            if len(domain) != 3: continue
            site = domain[1]
            if site in ignore: continue
            if not os.path.isdir(site):
                os.mkdir(site)
            os.chdir(site)
            if len(path) < 4: continue
            issue = path[3]
            if not os.path.isdir(issue):
                os.mkdir(issue)
            os.chdir(issue)
            if os.path.isdir(mode):
                os.chdir("..")
                os.chdir("..")
                continue
            os.mkdir(mode)
            os.chdir(mode)
            link = urllib.urlopen(href)
            page = link.read()
            print site, issue
            urls = re.findall(pattern5, page)
            for url in urls:
                parsed = urlparse.urlparse(url)
                path = parsed[2].split("/")
                file = path[-1]
                name, ext = os.path.splitext(file)
                if ext == ".wmv":
                    urllib.urlretrieve(url, file)
                    print file,

            print
            os.chdir("..")
            os.chdir("..")
            os.chdir("..")
            sys.exit(0)

print "no new issues"
sys.exit(1)