import pickle import pprint import datetime ''' 'clutch2data' reads the pickle file which was created by 'clutch' and generates various output data files. ''' NO_RELEASES = "{0} has Signing Keys with either no or invalid releases" messages = { '00':('/guides/mentor','add_to_incubation_summary_file','Duplicate entry in podlings.xml',True), '01':('/guides/mentor','add_to_incubation_summary_file','No project LDAP',True), '02':('/guides/mentor','add_to_incubation_summary_file','Project metadata missing in content/podlings.xml',True), '05':('/guides/mentor','add_to_incubation_summary_file','Reporting group is either missing or a multiple',True), '06':('/guides/mentor','add_to_incubation_summary_file','Reporting group is missing',True), '07':('/guides/mentor','initialize_podling_status_page','Status file is missing',True), '08':('/guides/mentor','add_to_incubation_summary_file','Start date is not a valid date',True), #'14':('/helper/bootstrap','email', 'Expected mailing list (incubator-{0}-{1}) is missing',False), '15':('/guides/mentor','request_mailing_lists',"Project does not have 'dev' mailing list",True), '16':('/guides/mentor','request_mailing_lists',"Project does not have 'commits' mailing list",False), '17':('/guides/mentor','git_migrations','No source code repository',False), '10':('/guides/mentor','self_service_requests','Project is missing its link to an Issue Tracker',False), '18':('/guides/sites','using_a_wiki_to_create_documentation','Project is missing its link to a Wiki',False), '09':('/guides/sites','podling_website_requirements','Apache Website is missing',False), '11':('/guides/sites','podling_website_requirements','Website is missing',False), '12':('/guides/releasemanagement','what_goes_into_a_release','Release distribution area is not setup',False), '13':('/guides/releasemanagement','what_goes_into_a_release','Release signing PGP KEYS file is missing',False), '41':('/guides/releasemanagement','what_goes_into_a_release','PGP KEYS file available without a distribution',False), '42':('/guides/releasemanagement','what_goes_into_a_release','Distribution is missing incubator/incubating in file name {0}',False), '43':('/guides/releasemanagement','what_goes_into_a_release','Signed distribution without a hash',True), '44':('/guides/releasemanagement','what_goes_into_a_release','Unsigned distributions',True), '45':('/guides/releasemanagement','what_goes_into_a_release','{0} is a large number of releases. Only current releases on current branches should be on dist.',False), '46':('/guides/releasemanagement','what_goes_into_a_release','Apache Releases cannot be validated without a KEYS file.',True), '31':('/guides/sites','podling_website_requirements', 'ASF Foundation link is missing',False), '32':('/guides/sites','podling_website_requirements', 'ASF Events link is missing',False), '33':('/guides/sites','podling_website_requirements', 'Apache License link is missing',False), '34':('/guides/sites','podling_website_requirements', 'Apache Thanks link is missing',False), '35':('/guides/sites','podling_website_requirements','Apache Security link is missing',False), '36':('/guides/sites','podling_website_requirements','Apache Sponsorship link is missing',False), '37':('/guides/sites','podling_website_requirements', 'Trademarks are missing',False), '38':('/guides/sites','podling_website_requirements', 'Copyright is missing',False), '39':('/guides/branding','disclaimers','Disclaimer is missing',False) #'03':('/helper/exit','graduated', 'Graduated, but some graduation steps are incomplete'), #'04':('/helper/exit','retired', 'Retired, but some retirement steps are incomplete') } def humanbytes(B): 'Return the given bytes as a human friendly KB, MB, GB, or TB string' B = float(B) KB = float(1024) MB = float(KB ** 2) # 1,048,576 GB = float(KB ** 3) # 1,073,741,824 TB = float(KB ** 4) # 1,099,511,627,776 if B < KB: return '{0} {1}'.format(B,'Bytes') elif KB <= B < MB: return '{0:.2f} KB'.format(B/KB) elif MB <= B < GB: return '{0:.2f} MB'.format(B/MB) elif GB <= B < TB: return '{0:.2f} GB'.format(B/GB) elif TB <= B: return '{0:.2f} TB'.format(B/TB) inputFile = open('clutch2.pkl', 'rb') projects = pickle.load(inputFile) inputFile.close() for k in projects: status = "content/clutch/{}.ad".format(k) f = open(status,'w') print("= Apache "+projects[k]['fullName']+" (Incubating)",file=f) print("Apache Incubator PMC",file=f) print(":jbake-type: simplepage",file=f) print(":jbake-status: published",file=f) print(":idprefix:",file=f) print(":imagesdir: /images/",file=f) print("",file=f) if projects[k]['sponsor'] != "Incubator": print(projects[k]['sponsor'],file=f) print("",file=f) print(" ".join(projects[k]['description'].split()),file=f) print("",file=f) print("* Resources: "+projects[k]['resource'],file=f) print("* Started: "+projects[k]['startDate']+"; Last Status Update: "+projects[k]['statusLastUpdated'],file=f) if projects[k]['reportingMonthly']: print("* Reporting: Monthly",file=f) else: print("* Reporting: "+projects[k]['reportingGroup'],file=f) try: print("* Committers: "+str(projects[k]['ldap']['numberCommitters']),file=f) if projects[k]['ldap']['numberPMCEquals']: print("* All Committers are PPMC members",file=f) else: print("* PPMC members: "+str(projects[k]['ldap']['numberPMCMembers']),file=f) except: print("* *No LDAP*",file=f) mentors = "* Mentors: " for m in projects[k]['mentors']: mentors += "{0} ({1}), ".format(m[0],m[1]) print(mentors[:-2],file=f) print("",file=f) if len(projects[k]['news']) > 0: print("== News",file=f) print("",file=f) for news in projects[k]['news']: print("* {0}".format(news),file=f) print("",file=f) print("== Resources",file=f) urls = projects[k]['urls'] if len(urls['www']) > 0: print("=== Project Website",file=f) print("",file=f) print("{0}".format(urls['www']),file=f) print("",file=f) print("=== Mailing Lists",file=f) print("",file=f) if len(projects[k]['hasMailListDev']) > 0: print("* Developer mailing list: {0}".format(projects[k]['hasMailListDev']),file=f) else: print("* No developer mailing list",file=f) if len(projects[k]['hasMailListCommits']) > 0: print("* Commits mailing list: {0}".format(projects[k]['hasMailListCommits']),file=f) else: print("* No commits mailing list",file=f) print("",file=f) if len(urls['tracker']) > 0: print("=== Issue Tracker",file=f) print("",file=f) print("{0}".format(urls['tracker']),file=f) print("",file=f) if len(urls['wiki']) > 0: print("=== Wiki",file=f) print("",file=f) print("{0}".format(urls['wiki']),file=f) print("",file=f) print("=== Repositories",file=f) print("",file=f) lsvn = len(urls['svn']) if lsvn > 0: print("{0}[Subversion]".format(urls['svn']),file=f) print("",file=f) i = len(urls['git']) j = len(projects[k]['repositories']) if i > 0 and j == 0: print("{0}[Git]".format(urls['git']),file=f) print("",file=f) elif j > 0: jj = 0 for l in projects[k]['repositories']: jj += 1 #if jj > 1 or lsvn > 0: # print("",file=f) desc = str(l[1]['description']) #commits = str(l[1]['commits']) last_update = str(l[1]['last_update_txt']) gityear = datetime.datetime.fromtimestamp(l[1]['last_update_int']).strftime('%Y') gitdate = datetime.datetime.fromtimestamp(l[1]['last_update_int']).strftime('%m/%d/%Y') if gityear < "1990": gitdate = "Fresh Repository" repos = l[0] print("==== {0}: {1}".format(jj,repos),file=f) print("",file=f) print("| *{0}{1}{2}[Gitbox]* | *{3}{4}/[Github]* | {5}".format( "https://gitbox.apache.org/repos/asf?p=",repos,".git;a=tree;hb=HEAD", "https://github.com/apache/",repos,desc),file=f) #print("-- Commits: {0}".format(commits),file=f) print("-- Updated: {0}".format(gitdate),file=f) print("",file=f) print("== Releases",file=f) print("",file=f) print("=== Current",file=f) print("",file=f) hasKeys = False url1 = urls['keys'] if len(url1) > 0: hasKeys = True print("*{0}[Signing Keys]*".format(url1),file=f) url1 = "/".join(url1.split("/")[:-1])+"/" print("",file=f) print("It is essential that you verify the integrity of release downloads. See https://www.apache.org/dyn/closer.cgi#verify[instructions here]",file=f) else: print("*No PGP Signing Keys*",file=f) jj = 0 fsize = 0 if len(projects[k]['releases']) > 0 and hasKeys: j = 1 for r in sorted(projects[k]['releases']): path = "{0}/{1}/{2}".format(k,projects[k]['releases'][r]['folder'],r) print("",file=f) print("==== {0}: {1}".format(j,r),file=f) print("",file=f) print("| *{0}{1}[Download]* | *{2}{3}.asc[Signature]* | *{4}{5}.{6}[Hash]* | {7}".format( "https://www.apache.org/dyn/closer.lua/incubator/", path, "https://downloads.apache.org/incubator/", path, "https://downloads.apache.org/incubator/", path, projects[k]['releases'][r]['hash'], r),file=f) print("-- Filesize: {0}".format(humanbytes(float(projects[k]['releases'][r]['size']))),file=f) print("-- Released: {0} by {1} in {2}".format( projects[k]['releases'][r]['dtm'], projects[k]['releases'][r]['user'], projects[k]['releases'][r]['revision']),file=f) j += 1 fsize += int(projects[k]['releases'][r]['size']) print("",file=f) print("Total size of all downloads = {0}".format(humanbytes(float(fsize))),file=f) elif hasKeys: print(NO_RELEASES.format(projects[k]['fullName']),file=f) print("",file=f) if len(projects[k]['zFixMeList']) > 0: print("== Errata",file=f) print("",file=f) print("Please investigate the following potential issues",file=f) print("",file=f) for l in projects[k]['zFixMeList']: msg = messages[l[0]] if msg[3]: print("* link:"+msg[0]+".html#"+msg[1]+"[*"+l[1]+"*]",file=f) else: print("* link:"+msg[0]+".html#"+msg[1]+"["+l[1]+"]",file=f) print("",file=f) print("The podling website scan does the best it can. Details are found https://whimsy.apache.org/pods/project/{0}[here]".format(k),file=f) print("",file=f) f.close(); # we count lines in stdout to compare for proper completion. print("{0} - {1}".format(k,status))