From 826408979db0e8e4675d51def6ce7dadd305cf9c Mon Sep 17 00:00:00 2001 From: Harishankar Date: Wed, 1 Dec 2010 12:21:23 +0530 Subject: [PATCH] Implemented additional files/folders adding * Implemented the copying of additional files and folders * Now checks whether required additional files exist in source directory. If found they are copied automatically. If not found, user is told to copy them manually. Required additional files are: search.py - search CGI script to be copied to cgi-bin/ star.gif and star-grey.gif - star rating images --- biaweb_exporter.py | 63 +++++++ generate_dialog.py | 37 ++++ generate_dialog.ui | 80 ++++++++- search.py | 395 ++++++++++++++++++++++++++++++++++++++++++ star-grey.gif | Bin 0 -> 898 bytes star.gif | Bin 0 -> 284 bytes ui_generate_dialog.py | 61 ++++++- 7 files changed, 619 insertions(+), 17 deletions(-) create mode 100755 search.py create mode 100644 star-grey.gif create mode 100644 star.gif diff --git a/biaweb_exporter.py b/biaweb_exporter.py index e824c08..4560181 100644 --- a/biaweb_exporter.py +++ b/biaweb_exporter.py @@ -3,6 +3,7 @@ import os import os.path +import sys import time import sqlite3 import string @@ -48,6 +49,63 @@ def html_format_rating (rating): rating_str = "".join (items) return rating_str +# function to copy additional files and folders to the destination path +def copy_files_folders (conf, files_to_copy, folders_to_copy): + # create the cgi-bin directory and try to copy search.py into the destination directory if possible + # otherwise user must copy it manually + search_script_path = os.path.join (sys.path[0], "search.py") + if os.path.exists (search_script_path): + try: + os.mkdir (os.path.join (conf[5], "cgi-bin")) + shutil.copy2 (search_script_path, os.path.join(conf[5], "cgi-bin")) + except IOError, OSError: + return False + + # try to copy the star rating images to destination directory if possible + # otherwise user must copy it manually + rating_img_star = os.path.join (sys.path[0], "star.gif") + rating_img_greystar = os.path.join (sys.path[0], "star-grey.gif") + if os.path.exists (rating_img_star): + try: + shutil.copy2 (rating_img_star, conf[5]) + except IOError, OSError: + return False + if os.path.exists (rating_img_greystar): + try: + shutil.copy2 (rating_img_greystar, conf[5]) + except IOError, OSError: + return False + + # additional files to copy + + # first copy files + # check if files to copy is not empty + if files_to_copy <> []: + for src, dest in files_to_copy: + # get full path from relative path in dest + full_dest = os.path.join (conf[5], dest) + try: + shutil.copy2 (src, full_dest) + except IOError, OSError: + return False + + # additional folders to copy + + # now copy the folders + if folders_to_copy <> []: + for src, dest in folders_to_copy: + # get full path from relative path in dest + full_dest = os.path.join (conf[5], dest) + try: + shutil.copytree (src, full_dest) + except IOError, OSError: + return False + + # finally return true + return True + + + # function to generate article pages def generate_article_pages (dbname, conf, templates, category_str, bestrated_str): # main template @@ -273,5 +331,10 @@ def generate_site (dbname, files_to_copy, folders_to_copy, search_type_full=True if ret == False: return False + # copy other files/folders into the destination path + ret = copy_files_folders (conf, files_to_copy, folders_to_copy) + if ret == False: + return False + # finally when all is successfully done return true return True diff --git a/generate_dialog.py b/generate_dialog.py index dc0448b..e236451 100644 --- a/generate_dialog.py +++ b/generate_dialog.py @@ -2,6 +2,7 @@ # Generate site dialog import PyQt4 +import sys import os import os.path import ui_generate_dialog @@ -12,9 +13,45 @@ class GenerateDialog (PyQt4.QtGui.QDialog, ui_generate_dialog.Ui_SiteGenerateDia PyQt4.QtGui.QDialog.__init__ (self, master) self.setupUi (self) + # update the status of required additional files to be copied to + # the destination directory and warn the user accordingly + self.set_required_files_status () + # set the database self.current_db = currentdb + # function to set the status message of the required additional files to be copied to + # the destination directory. If any required additional file is missing in the script + # path then warn the user to copy those files manually to the destination directory + def set_required_files_status (self): + search_script_path = os.path.join (sys.path[0], "search.py") + star_image_path = os.path.join (sys.path[0], "star.gif") + stargrey_image_path = os.path.join (sys.path[0], "star-grey.gif") + + # if search.py exists in script directory + if os.path.exists (search_script_path): + self.status_search_py.setText ('will be \ + automatically copied to destination dir') + else: + self.status_search_py.setText ('cannot be found.\ + You must copy it manually to cgi-bin') + + # if star.gif exists in script directory + if os.path.exists (star_image_path): + self.status_star_gif.setText ('will be \ + automatically copied to destination dir') + else: + self.status_star_gif.setText ('Cannot be found. \ + You must copy it manually to destination dirwill be \ + automatically copied to destination dir') + else: + self.status_stargrey_gif.setText ('cannot be found. \ + You must copy it manually to destination dirGenerate Website - + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> @@ -34,16 +34,80 @@ p, li { white-space: pre-wrap; } </style></head><body style=" font-family:'Sans'; font-size:10pt; font-weight:400; font-style:normal;"> <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600; color:#55007f;">Important Information</span></p> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">This will generate your website in the destination directory specified in the site configuration dialog. The entire directory tree will be re-created from scratch and any manual changes you made to the generated files will be gone!</p> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"> </p> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-style:italic;">You need to re-copy the additional files/folders you require in the destination folder (e.g. the rating (star) image files or directories, the search.py CGI script) every time you generate the site. Specify the files/folders to copy to generated site.</span></p></body></html> +<p style=" margin-top:8px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">This will generate your website in the destination directory configured. The directory tree will be re-created from scratch and any manual changes you made to the generated files will be destroyed!</p> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-style:italic;">You need to re-copy the additional files/folders you require in the destination folder (e.g. image directories) every time you generate the site. Specify the files/folders to copy to generated site.</span></p></body></html> true - + + + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> +<html><head><meta name="qrichtext" content="1" /><style type="text/css"> +p, li { white-space: pre-wrap; } +</style></head><body style=" font-family:'Sans'; font-size:10pt; font-weight:400; font-style:normal;"> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Required additional files</span></p></body></html> + + + + + + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> +<html><head><meta name="qrichtext" content="1" /><style type="text/css"> +p, li { white-space: pre-wrap; } +</style></head><body style=" font-family:'Sans'; font-size:10pt; font-weight:400; font-style:normal;"> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">search.py</span> (search CGI script):</p></body></html> + + + + + + + status + + + + + + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> +<html><head><meta name="qrichtext" content="1" /><style type="text/css"> +p, li { white-space: pre-wrap; } +</style></head><body style=" font-family:'Sans'; font-size:10pt; font-weight:400; font-style:normal;"> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">star.gif</span> (star rating image):</p></body></html> + + + + + + + status + + + + + + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> +<html><head><meta name="qrichtext" content="1" /><style type="text/css"> +p, li { white-space: pre-wrap; } +</style></head><body style=" font-family:'Sans'; font-size:10pt; font-weight:400; font-style:normal;"> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">star-grey.gif</span> (star rating image):</p></body></html> + + + + + + + status + + + + 0 @@ -134,7 +198,7 @@ p, li { white-space: pre-wrap; } - + Search index type @@ -163,7 +227,7 @@ p, li { white-space: pre-wrap; } - + &Generate site @@ -173,7 +237,7 @@ p, li { white-space: pre-wrap; } - + &Close diff --git a/search.py b/search.py new file mode 100755 index 0000000..cc5ab6e --- /dev/null +++ b/search.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python + +# hari's search engine front-end +# this program queries the search index created by hari's search index creator application + +import string +import cPickle +import cgi +import os +import os.path +import time + +# ----------------------------------------------------------- +# configuration settings +# ----------------------------------------------------------- + +# script start time +strttime = time.time () + +# path to the search database index file - edit as necessary +search_db_path = "searchwords.idx" + +# path to file containing the "stop words" words which should be excluded from the query +stopwords_list = """$ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +a +able +about +after +again +all +almost +already +also +although +am +an +and +another +any +are +are +around +as +at +b +based +be +because +been +before +being +between +both +bring +but +by +c +came +can +com +come +comes +could +d +did +do +does +doing +done +e +each +eight +else +etc +even +every +f +five +for +four +from +g +get +gets +getting +go +going +got +h +had +has +have +he +he +her +here +him +himself +his +how +however +href +http +i +if +in +including +into +is +it +it +its +j +just +k +kb +know +l +like +looks +m +mailto +make +making +many +may +mb +me +means +might +more +more +most +move +mr +much +must +my +n +need +needs +never +nice +nine +no +not +now +o +of +often +oh +ok +on +on +one +only +or +org +other +our +out +over +own +p +piece +q +r +rather +re +really +s +said +same +say +says +see +seven +several +she +should +since +single +six +so +so +some +something +still +stuff +such +t +take +ten +than +that +the +their +them +them +then +there +there +these +they +they +thing +things +this +those +three +through +to +too +took +two +u +under +up +us +use +used +using +usual +v +ve +very +via +w +want +was +way +we +we +well +were +what +when +where +whether +which +while +whilst +who +why +will +with +within +would +x +y +yes +yet +you +your +z""".split ("\n") + +# first print the header and the HTML code +print "Content-type: text/html; charset=utf-8" +print """ + + + Search Results + + +

Search results

+
+""" + +# check whether the search database exists at all +if not os.path.exists (search_db_path) and not os.path.isfile (search_db_path): + print "Search database file seems to be missing. Cannot search." + print "" + print "" + sys.exit () + +frm = cgi.FieldStorage () + +# if the form has not been submitted, then ask for it to be submitted +if not frm.has_key ("fromsearch") or frm.getfirst("query") == "" : + print "No search query defined." + print "" + print "" +else: + # get the query + raw_query = frm.getfirst ("query").strip ().lower () + # get the mode of the query - AND or NOT + mode = int (frm.getfirst ("criteria")) + + # split the terms + raw_query_list = raw_query.split () + + # get the database of search terms + try: + search_db = cPickle.load (open (search_db_path, "rb")) + except: + print "Error reading search database: might not valid." + + # if the number of search words is too much + if len (raw_query_list) > 5: + print "

Too many terms. Try fewer search terms (avoid common words like a, and, or, in).

" + print "" + print "" + sys.exit () + + + # display the list of search terms + print "Search terms: ", + + # search words set + search_words = set () + + # add the search words to the set + for raw_word in raw_query_list: + # strip the raw word of all non-alphabetic characters at the beginnning and end + word = raw_word.strip (":;?!_<>,.+-\"'=`!@#$%^&*()[]{}/= \n\r\t") + + # only if the word is not in the stopwords list add it to the query + if not word in stopwords_list: + print "" + word + ", ", + search_words.add (word) + # just display the word with a strike-through to indicate its non-inclusion + else: + print "" + word + ", ", + + # if there are no specific terms found + if len(search_words) == 0: + print "

Search terms are too broad and commonly used words. Try searching with at least one specific term.

" + print "" + print "" + sys.exit () + + # get the search results + search_results = set () + flagfirst = True + + # print search mode + print "

Search mode: ", + if (mode == 1): + print "documents containing all search terms

" + else: + print "documents containing any of the search terms

" + + # get the words found in the database + for word in search_words: + # if the word is found + if search_db.has_key (word): + if (flagfirst == True): + search_results = search_db[word] + flagfirst = False + else: + # AND mode + if (mode == 1): + search_results = search_results & search_db[word] + # Assume OR mode + else: + search_results = search_results | search_db[word] + # if word is not found and the mode is AND, then return empty result + else: + if (mode == 1): + search_results.clear () + break + + # display the results + + # if no results are found + if len (search_results) == 0: + print "

No pages matched your query.

" + else: + # display the result + print "

Documents found

" + print "
    " + for title, url in search_results: + print '
  • ' + title + '' + print "
" + +# print the script execution time +endtime = time.time () +dif = endtime - strttime + +print "
" +print "Script execution time: ", +print dif, +print "sec" +print "" +print "" diff --git a/star-grey.gif b/star-grey.gif new file mode 100644 index 0000000000000000000000000000000000000000..ca97b2d48f457f4824bfd3500e59a8781b71fc76 GIT binary patch literal 898 zcmV-|1AY8QNk%w1VGIBZ0OtSz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7 zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}? zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z} zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5( zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p zxVX5vxw*Q!y1To(yu7@dCU$jHda z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1lt)=I7_<=;-L_>FMg~>g((4 z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg= z{r&#_{{R2~A^s6Va%Ew3Wn>_CX>@2HM@dak03rDV0SW;B04x9i000aC3jhEJN&x=} z97wRBKtzQM5)39x;lOR%44w&At5~~sw?NtYBCt%Uiy+gPLPIbP$abU*Mk#mEn?hFH Y3Q#t@=I1k+|#bD2%!vF*zKQXXb9++R?p~EG~ zt!%hBkk48sSuy!$kK2snsR}9{>mFP<;^5NYF1VPX\n" "

Important Information

\n" -"

This will generate your website in the destination directory specified in the site configuration dialog. The entire directory tree will be re-created from scratch and any manual changes you made to the generated files will be gone!

\n" -"

\n" -"

You need to re-copy the additional files/folders you require in the destination folder (e.g. the rating (star) image files or directories, the search.py CGI script) every time you generate the site. Specify the files/folders to copy to generated site.

", None, QtGui.QApplication.UnicodeUTF8)) +"

This will generate your website in the destination directory configured. The directory tree will be re-created from scratch and any manual changes you made to the generated files will be destroyed!

\n" +"

You need to re-copy the additional files/folders you require in the destination folder (e.g. image directories) every time you generate the site. Specify the files/folders to copy to generated site.

", None, QtGui.QApplication.UnicodeUTF8)) + self.label_4.setText(QtGui.QApplication.translate("SiteGenerateDialog", "\n" +"\n" +"

Required additional files

", None, QtGui.QApplication.UnicodeUTF8)) + self.label_5.setText(QtGui.QApplication.translate("SiteGenerateDialog", "\n" +"\n" +"

search.py (search CGI script):

", None, QtGui.QApplication.UnicodeUTF8)) + self.status_search_py.setText(QtGui.QApplication.translate("SiteGenerateDialog", "status", None, QtGui.QApplication.UnicodeUTF8)) + self.label_6.setText(QtGui.QApplication.translate("SiteGenerateDialog", "\n" +"\n" +"

star.gif (star rating image):

", None, QtGui.QApplication.UnicodeUTF8)) + self.status_star_gif.setText(QtGui.QApplication.translate("SiteGenerateDialog", "status", None, QtGui.QApplication.UnicodeUTF8)) + self.label_7.setText(QtGui.QApplication.translate("SiteGenerateDialog", "\n" +"\n" +"

star-grey.gif (star rating image):

", None, QtGui.QApplication.UnicodeUTF8)) + self.status_stargrey_gif.setText(QtGui.QApplication.translate("SiteGenerateDialog", "status", None, QtGui.QApplication.UnicodeUTF8)) self.label_2.setText(QtGui.QApplication.translate("SiteGenerateDialog", "Additional files to add to destination", None, QtGui.QApplication.UnicodeUTF8)) self.addfile.setText(QtGui.QApplication.translate("SiteGenerateDialog", "&Add file", None, QtGui.QApplication.UnicodeUTF8)) self.removefile.setText(QtGui.QApplication.translate("SiteGenerateDialog", "&Remove", None, QtGui.QApplication.UnicodeUTF8)) -- 2.20.1