my first python shit

From cm3l1k1, 5 Years ago, written in Python, viewed 452 times.
URL http://paste.security-portal.cz/view/bb96023c Embed
Download Paste or View Raw
  1. #!/usr/bin/env python
  2. #
  3. # desc: Script for mirroring stage3 and portage files
  4. #
  5. # -*- coding: utf-8 -*-
  6.  
  7. import os
  8. import urllib2
  9. import math
  10. import hashlib
  11. import cgitb
  12. cgitb.enable()
  13.  
  14.  
  15. #
  16. # VARIABLES
  17. #
  18. temp_path = os.listdir(os.getcwd()) + "/tmp/"
  19. current_path = os.listdir(os.getcwd())
  20. BASE_URL  = "http://distfiles.gentoo.org/"
  21. PORTAGE_FILE = "snapshots/portage-latest.tar.bz2"
  22. PORTAGE_LASTMOD_FILE = "lastmod.txt"
  23. STAGE3_FILE = "releases/amd64/autobuilds/latest-stage3-amd64-hardened.txt"
  24.  
  25. #
  26. # FUNCTIONS
  27. #
  28.  
  29. # remove lines with # from file
  30. def processConf(s):
  31.         "Remove #comments from string."
  32.         s = map(lambda x: x.split("#")[0], s.splitlines())
  33.         return filter(lambda x: x.strip() != "", s)
  34.  
  35. # create directory structure if not exist
  36. def createIfNotExists(path):
  37.         if not os.path.exists(path):
  38.                 os.makedirs(path)
  39.  
  40. # download files in chunks
  41. def downloadChunks(url):
  42.  
  43.     baseFile = os.path.basename(url)
  44.     tempFile = temp_path + baseFile
  45.     # read mtime of file what we already have
  46.     lastmod = "0"
  47.     if os.path.isfile(tempFile + ".lastmod"):
  48.        fh = open(tempFile + ".lastmod")
  49.        lastmod = fg.read()
  50.        fh.close()
  51.    
  52.     #os.umask(0002)
  53.     try:
  54.         file = os.path.join(temp_path,baseFile)
  55.  
  56.         req = urllib2.urlopen(url)
  57.         total_size = int(req.info().getheader('Content-Length').strip())
  58.         file_lastmod = int(req.info().getheader('Last-Modified').strip())
  59.         # if we already have file with this mtime, do nothing!
  60.         if lastmod == file_lastmod:
  61.            print "XXX JAK KUA ZRUSIM BEH FUNKCE?"
  62.         downloaded = 0
  63.         CHUNK = 256 * 10240
  64.         with open(file, 'wb') as fp:
  65.             while True:
  66.                 chunk = req.read(CHUNK)
  67.                 downloaded += len(chunk)
  68.                 #print math.floor( (downloaded / total_size) * 100 )
  69.                 if not chunk: break
  70.                 fp.write(chunk)
  71.     except urllib2.HTTPError, e:
  72.         print "HTTP Error:",e.code , url
  73.         return False
  74.     except urllib2.URLError, e:
  75.         print "URL Error:",e.reason , url
  76.         return False
  77.     # if there is newer file, save new mtime
  78.     if lastmode != file_lastmode:
  79.         fh = open(temp_file + ".lastmod", "wt")
  80.         fh.write(file_lastmod)
  81.         fh.close()
  82.  
  83.     return file
  84.  
  85. # make MD5 sum
  86. def md5_for_file(f, block_size=2**20):
  87.     md5 = hashlib.md5()
  88.     while True:
  89.         data = f.read(block_size)
  90.         if not data:
  91.             break
  92.         md5.update(data)
  93.     return md5.hexdigest()
  94.  
  95. # make SHA512 sum
  96. def sha512_for_file(f, block_size=2**20):
  97.     sha512 = hashlib.sha512()
  98.     while True:
  99.         data = f.read(block_size)
  100.         if not data:
  101.             break
  102.         sha512.update(data)
  103.     return sha512.hexdigest()
  104.  
  105.  
  106. #
  107. # MAIN
  108. #
  109.  
  110. # print for CGI output
  111. print "Content-Type: text/html;charset=utf-8"
  112. print
  113.  
  114. #
  115. # PROCESS stage3 amd64 file
  116. #
  117.  
  118. downloadChunks(BASE_URL + STAGE3_FILE)
  119. STAGE3_FILE_TMP = tempFile
  120.  
  121. # read content of file to find path for latest version
  122. STAGE3_LATEST = processConf(STAGE3_FILE_TMP)
  123. if 1:
  124.         STAGE3_URI = os.path.dirname(STAGE3_FILE)
  125.         # create dir structure if not exist
  126.         createIfNotExists(STAGE3_URI)
  127.         # download latest files
  128.         STAGE3_LATEST_FILE = BASE_URL + STAGE3_URI + STAGE3_LATEST
  129.         downloadChunks(STAGE3_LATEST_FILE)
  130.         STAGE3_LATEST_FILE_TMP = tempFile
  131.         # make sha512 hash
  132.         STAGE3_SHA512 = sha512_for_file(tempFile)
  133.         # download control hash
  134.         downloadChunks(STAGE3_LATEST_FILE + ".DIGESTS")
  135.         STAGE3_SHA512_CONTROL_TMP = tempFile
  136.         STAGE3_SHA512_CONTROL = processConf(tempFile)
  137.  
  138. if STAGE3_SHA512 == STAGE3_SHA512_CONTROL:
  139.         # move files from temp to appropriate folders
  140.         os.rename(STAGE3_FILE_TMP,current_path + STAGE3_FILE)
  141.         os.rename(STAGE3_LATEST_FILE_TMP,current_path + STAGE3_URI + STAGE3_LATEST)
  142.         os.rename(STAGE3_SHA512_CONTROL_TMP,current_path + STAGE3_URI + STAGE3_LATEST + ".DIGESTS")
  143.         print "--- Stage3 completed"
  144.  
  145. # XXX jak se kurva dela else?
  146.  
  147. #
  148. # PROCESS portagefile
  149. #
  150.         # download portage and checksum
  151.         downloadChunks(BASE_URL + PORTAGE_FILE)
  152.         PORTAGE_FILE_TMP = tempFile
  153.         PORTAGE_HASH = md5_for_file(tempFile)
  154.         downloadChunks(BASE_URL + PORTAGE_FILE + ".md5sum")
  155.         PORTAGE_MD5_CONTROL_TMP = tempFile
  156.        
  157.         # XXX POUZIT REGEX NEBO SH PRO SPUSTENI MD5SUM
  158.         # pripadne funkci typu - if contain string, then ok
  159.        
  160.         # create directory structure
  161.         createIfNotExists(os.path.dirname(PORTAGE_FILE))
  162.         # move files from temp to appropriate folders
  163.         os.rename(PORTAGE_FILE_TMP,current_path + PORTAGE_FILE)
  164.         os.rename(PORTAGE_MD5_CONTROL_TMP,current_path + PORTAGE_FILE + ".md5sum")
  165.         print "--- Portage completed"
  166.  
  167.  
  168. print "###-### All Done ###-###"

Reply to "my first python shit"

Here you can reply to the paste above