ftpsync

From cm3l1k1, 5 Years ago, written in Python, viewed 379 times.
URL http://paste.security-portal.cz/view/5ea4c35a Embed
Download Paste or View Raw
  1. #!/usr/bin/env python
  2. #
  3. #: info: Mirroring specific files and folder structure from FTP server
  4. #: author: Martin Cmelik & script examples over Internet
  5. #
  6. #
  7. #
  8. # -*- coding: utf-8 -*-
  9.  
  10. import os
  11. import time
  12. import sys
  13. import ftplib
  14. import glob
  15. from ftplib import FTP
  16.  
  17. ftp=FTP("ftp.fi.muni.cz")
  18. ftp.login("anonymous","info@securix.org")
  19.  
  20. #
  21. # FUNCTIONS
  22. #
  23.  
  24. # remove lines with # from file
  25. def processConf(s):
  26.         "Remove #comments from string."
  27.         s = map(lambda x: x.split("#")[0], s.splitlines())
  28.         return filter(lambda x: x.strip() != "", s)
  29.  
  30. def downloadFiles(path,destination,filematch):
  31.     try:
  32.         # sync paths on ftp and local
  33.         ftp.cwd(path)
  34.         if not os.path.isdir(destination):
  35.             os.makedirs(destination)
  36.             print ("--- Created dir: "+destination)
  37.         os.chdir(destination)
  38.     except OSError:
  39.         # folder already exist
  40.         pass
  41.     except ftplib.error_perm:
  42.         # invalid entry (ensure input form: "/dir/folder/something/")
  43.         print ("--- Error: could not change to "+path)
  44.         sys.exit("ending session")
  45.  
  46.     # get file list
  47.     filelist = ftp.nlst(filematch)
  48.     for ftpfile in filelist:
  49.         # check if file already exist
  50.         if os.path.isfile(destination+ftpfile):
  51.             # check local file size
  52.             local_size = os.path.getsize(destination+ftpfile)
  53.             # check remote file size
  54.             ftp.sendcmd("TYPE i")
  55.             remote_size = ftp.size(ftpfile)
  56.             if local_size != remote_size:
  57.                 lf = open(os.path.join(destination,ftpfile),"wb")
  58.                 ftp.retrbinary("RETR "+ftpfile, lf.write, 8*1024)
  59.                 print ("--- Changed file: "+ftpfile)
  60.                 return
  61.             else:
  62.                 print ("--- Same file: "+ftpfile)
  63.                 return
  64.         # if it is a new file, download it
  65.         lf = open(os.path.join(destination,ftpfile),"wb")
  66.         ftp.retrbinary("RETR "+ftpfile, lf.write, 8*1024)
  67.         print ("--- New file: "+ftpfile)
  68.     return
  69.  
  70. # function goes through path and delete files older than 60 days
  71. # or empty folders
  72. def deleteOldFiles(path):
  73.     if len(os.listdir(path)) == 0:
  74.         print ("--- Deleting empty folder: "+path)
  75.         os.rmdir(path)
  76.         return
  77.     now = time.time()
  78.     for oldfile in os.listdir(path):
  79.         # delete files older then 60 days
  80.         if os.stat(oldfile).st_mtime < (now - 60*86400):
  81.             if os.path.isfile(oldfile):
  82.                 os.remove(os.path.join(path,oldfile))
  83.             # recursively go through path
  84.             elif os.path.isdir(oldfile):
  85.                 deleteOldFiles(oldfile)
  86.             print ("--- Deleted file: "+oldfile)
  87.     return
  88.    
  89. #
  90. # DOWNLOAD FILES
  91. #
  92.  
  93. # sync Portage files
  94. source="/pub/linux/gentoo/releases/snapshots/current/"
  95. dest="./releases/snapshots/current/"
  96. filematch="portage-latest.tar.bz*"
  97. downloadFiles(source,dest,filematch)
  98.  
  99. # sync amd64 Stage3 LATEST file
  100. source="/pub/linux/gentoo/releases/amd64/autobuilds/"
  101. dest="./releases/amd64/autobuilds/"
  102. filematch="latest-stage3-amd64-hardened.txt"
  103. downloadFiles(source,dest,filematch)
  104.  
  105. LATEST_STAGE3 = processConf(os.path.join(dest,filematch))
  106.  
  107. # sync amd64 Stage3 files
  108. source=("/pub/linux/gentoo/releases/amd64/autobuilds/" + LATEST_STAGE3)
  109. dest=("./releases/amd64/autobuilds/" + LATEST_STAGE3)
  110. filematch="stage3-amd64-hardened-*"
  111. downloadFiles(source,dest,filematch)
  112.  
  113. # sync x86 Stage3 LATEST file
  114. source="/pub/linux/gentoo/releases/x86/autobuilds/"
  115. dest="./releases/x86/autobuilds/"
  116. filematch="latest-stage3-i686-hardened.txt"
  117. downloadFiles(source,dest,filematch)
  118.  
  119. LATEST_STAGE3 = processConf(os.path.join(dest,filematch))
  120.  
  121. # sync x86 Stage3 files
  122. source=("/pub/linux/gentoo/releases/x86/autobuilds/" + LATEST_STAGE3)
  123. dest=("./releases/x86/autobuilds/" + LATEST_STAGE3)
  124. filematch="stage3-i686-hardened-*"
  125. downloadFiles(source,dest,filematch)
  126.  
  127. # make cleanup on server
  128. deleteOldFiles("./releases/")
  129.  

Reply to "ftpsync"

Here you can reply to the paste above