Python notes: FTP client, multi thread

Example for Python FTP client download in multi threads

from threading import Thread
from Queue import Queue
from ftplib import FTP
import datetime
import os
import time

FTP_Address = "your.FTP.Server"
FTP_Username = "admin"
FTP_Password = "xxxxxxxx"

Root_Folder_Name = "Recordings"
Cam_Names = ["Livingroom", "Kitchen", "Aisle", "Room", "Desk"]

# Check if the date folder exist on user's Desktop
Local_Desktop_Path = os.path.join(os.path.expanduser("~"), "Desktop")
Local_Date_Path = os.path.join(Local_Desktop_Path, Date_To_Downalod)
if not os.path.exists(Local_Date_Path):
    os.makedirs(Local_Date_Path)

# Create FTP connection
ftp = FTP(FTP_Address)
ftp.login(FTP_Username, FTP_Password)

# Create queue for threads
queue = Queue()
NUM_THREADS = 5

Local_Cam_Folder_Path = os.path.join(Local_Date_Path, Cam_Names[3])
if not os.path.exists(Local_Cam_Folder_Path):
    os.makedirs(Local_Cam_Folder_Path)
 
ftp.cwd("/" + Root_Folder_Name + "/" + Cam_Names[3])
if Date_To_Downalod not in ftp.nlst():
    print "%s: \t There is no %s picture, skip..." % (Cam_Names[3], Date_To_Downalod)
    return
ftp.cwd(Date_To_Downalod)

# get filenames within the directory
HourFolders = ftp.nlst()

# get local file list
Local_filenames = os.listdir(Local_Cam_Folder_Path)

for HourFolder in HourFolders:
    ftp.cwd(HourFolder)
    # get filenames within the directory
    FTP_filenames = ftp.nlst()
    # filter existing files
    filenames = set(FTP_filenames) - set(Local_filenames)
    for filename in filenames:
        # Not to create hour folder
        local_filename = os.path.join(Local_Cam_Folder_Path, filename)
        # check if file exist
        if not os.path.isfile(local_filename):
            queue.put(["/"+Root_Folder_Name+"/"+Cam_Names[3]+"/"+Local_Folder+"/"+HourFolder, filename, local_filename])
        i=i+1
        print "\r%s:\t %d files to be downloaded" % (Cam_Names[3], i),
    ftp.cwd("..")

# Close FTP connection
ftp.quit()

# Kick off threads to download
# Create threads
threads = map(lambda i: Thread(target=Download_Worker), xrange(NUM_THREADS))

# Make all threads start their activities
map(lambda th: th.start(), threads)  
while not queue.empty():
    print "\rDownloading cam pictures, remaining:\t %d\t" % queue.qsize(),
    time.sleep(2) # delays for 2 seconds
print "\rDownloading cam pictures, remaining:\t 0\t"

# block until all threads are terminated
map(lambda th: th.join(), threads)   


#############################################
# Download Worker                           #
#############################################
def Download_Worker():
    ftp = FTP(FTP_Address)
    ftp.login(FTP_Username, FTP_Password)
    while not queue.empty():    # exit when queue is clear
        temp_list = queue.get()
        FTP_Path=temp_list[0]
        FTP_Filename=temp_list[1]
        Local_File_Path=temp_list[2]

        ftp.cwd(FTP_Path)
        file = open(Local_File_Path, 'wb')
        ftp.retrbinary('RETR '+ FTP_Filename, file.write)
        file.close()
    ftp.quit()