@file rst2leo.py

@ @rst-options
code-mode = True
number-code-lines = False
show-options-doc-parts = True

<<rst2leo declarations>>


<<rst2leo declarations>>

#all the allowable underline characters
valid_underline_characters = ['!','"','#','$','%','&',"'",'(',')','*','+',

class ParseReST

class ParseReST:
    """Processes a chunks of ReST, creating a list of nodes/sections


def __init__(self, input):

    """Initialize document level variables
    input is a list of strings or a string.
    If it's a string, it's split.

    if type(input) == type('string') or \
    type(input) == type(u'string'):
        self.lines = input.split("\n")
        self.lines = input

    self.index = 0

    # for each section gather title, contents and underline character
    # over-under titles are indicated by
    # 2 character strings for underline_character
    # the initial section is root
    self.section = {'title':'root', 'contents':[], 'underline_character':'root'}
    # the list of sections
    self.sections = []


def _isCharacterLine(self):
    """Determine if the current line consists of only
    valid underline characters
    line = self.lines[self.index]
    is_character_line = False
    if len(line) > 0:
        if line[0] in valid_underline_characters:
            c = line[0]
            for char in line:
                if char == c:
                    is_character_line = True
                    is_character_line = False
                    #get out of the loop
                    #otherwise error if 1st and last are characters
        return False
    return is_character_line


def _isTransition(self):
    """self.index is pointing to a character line
    if there are blank lines on either side, this is a transition

    current = self.lines[self.index]
    prev = self.lines[self.index - 1]
    next = self.lines[self.index + 1]

    return len(prev) == 0 and len(next) == 0


def _isUnderline(self):
    """self.index is pointing to a character line
    if two lines back is a blank line, the previous line
    is not longer than this, we have an underline

    current = self.lines[self.index].strip()
    prev = self.lines[self.index - 1].strip()
    prevprev = self.lines[self.index - 2].strip()

    return len(prev) > 0 and \
    len(prev) <= len(current) and \
    len(prevprev) == 0


def _isUnderOverline(self):
    """self.index is pointing at a character line
    if there is a line not longer than this
    followed by a character line like this,
    we have an UnderOverline

    current = self.lines[self.index].strip()
    next = self.lines[self.index + 1].strip()
    #the last line may be a character line
        nextnext = self.lines[self.index + 2]
    except IndexError:
        return False

    return (nextnext == current) and (len(next) > 0) \
    and len(next) <= len(current)


def _isSectionHead(self):
    """The current line is a character line,
    is this a section heading?
    # save typing with aliases
    current = self.lines[self.index]
    prev = self.lines[self.index - 1]
    next = self.lines[self.index + 1]

    # a transition has a blank line before and after
    if  self._isTransition():
        return False

    # underline section heading
    if self._isUnderline():
        # previous to discovering the underline, we appended
        # the section title to the current section.
        # Remove it before closing the section
        self.section['underline_character'] = current[0]
        self.section['title'] = prev
        # step index past this line
        self.index += 1
        return True

    # over-under section heading
    if self._isUnderOverline():
        self.section['underline_character'] = current[0:2]
        # leading whitespace is allowed in over under style, remove it
        self.section['title'] = next.strip()
        # step index past overline, section title, and underline
        self.index += 3
        return True

        raise Exception ("Error in foundSectionHead()")


def _closeCurrentSection(self):
    """We have a section title, which ended the previous
    section. Add this section to nodes, and start the next
    self.section = {'title':'', 'contents':[], 'underline_character':''}


def _insertTitle(self, uc, isSubTitle = False):
    """Inserting a title consists of merging section[1],
    the first section, into section[0], the root.
    This works the same for title and subtitle, since
    merging title deletes section[1], making the subtitle

    The 'isSubTitle' parameter differentiates between title and subtitle
    title = self.sections[1]['title']

    if not isSubTitle:
        self.sections[0]['title'] = title

    # extend the charline and pad the title
    charline = (len(title) * uc[0]) + (4 * uc[0])
    title = '  ' + title


    # append each line, not the list of lines
    for line in self.sections[1]['contents']:

    del self.sections[1]


def _fixupSections(self):
    """Make corrections to the list of sections
    to reflect the syntax for 'Title' and 'Subtitle'

    If the first section heading is a unique over/under
    it is a title, and should stay in the root section.

    If the second section heading is a unique over/under
    it is a subtitle and should remain in the root section.

    def isUnique(uc, start):
        index = start
        while index < len(self.sections):
            if self.sections[index]['underline_character'] == uc:
                return False
            index += 1
        return True

    # self.sections[0] is root, a special case
    underline_first = self.sections[1]['underline_character']
    if len(underline_first) > 1:
        if isUnique(underline_first, 2):
            # the section head is the document title and must
            # be added to the root section
    if len(self.sections) > 2:
        underline_second = self.sections[2]['underline_character']
        if len(underline_second) > 1:
            if isUnique(underline_second, 3):
                # the section head is the document subtitle and must
                # be added to the root section
                self._insertTitle(underline_second, True)


def processLines(self):
    """Loop through the lines of ReST input, building a list
    of sections. A section consists of::
    line_count = len(self.lines)

    while self.index < line_count:
        if self._isCharacterLine() and self._isSectionHead():
            # isCharacterLine() and isSectionHead() do all the housekeeping
            # required. This doesn't look like good style, but I'm not
            # sure how this should be written.
            self.index += 1

    if len(self.sections) > 1:
        if len(self.sections[0]['underline_character']) > 1:
    return self.sections

class BuildLeo

class BuildLeo:
    """Create a tree of nodes in a Leo file using a list of sections"""


def __init__(self, nodes, position, leoGlobals):
    """the nodes paramater is returned by ParseReST.processLines
    It is a list of dictionaries consisting of
    underline_character, title, contents
    self.nodes = nodes
    self.p = position
    self.c = position.c
    self.g = leoGlobals

    # self.levels is a dictionary, the keys
    # are underline_character and the value is the
    # last Leo node created at that level
    self.levels = {}

    # self.underline_characters is a list of the underline characters
    # in the order of levels. The first is always 'root'
    self.underline_characters = ['root',]


def _setRootNodeHeadString(self, rootstring):

    rst3_config = getConfig(rootstring, '.. rst3:')
    rst3_filename = None

    if rst3_config:
        rst3_filename = getConfig(rst3_config, 'filename:')

    if rst3_filename:
        return '@rst %s' % rst3_filename


def _dedent(self, text):
    current = self.p
    d = self.g.scanDirectives(self.c,current) # Support @tab_width directive properly.
    tab_width = d.get("tabwidth",self.c.tab_width)
    result = [] ; changed = False

    for line in text:
        i, width = self.g.skip_leading_ws_with_indent(line,0,tab_width)
        s = self.g.computeLeadingWhitespace(width-abs(tab_width),tab_width) + line[i:]
        if s != line: changed = True

    return result


def _fixCodeNodes(self):
    """The main block loops through nodes calling isCodeNode on each
    node. isCodeNode finds and removes markup added to code nodes
    by Leo.
    code_node_tag = '**code**:'
    code_block_tag = '.. code-block::'
    class_code_tag = '.. class:: code'
    comment_tag = '..'
    literal_block_tag = '::'

    tags = [code_node_tag, code_block_tag, class_code_tag,
            comment_tag, literal_block_tag]

    def isCodeNode(data, index):
        is_code_node = False
        index = 0
        num_lines = len(data)

        while index < num_lines:
            line = data[index]
            if line.find(code_node_tag) > -1 or \
            line.find(code_block_tag) > -1:
                is_code_node = True
            index += 1

        return is_code_node

    index = 0
    num_nodes = len(self.nodes)
    while index < num_nodes:
        data = self.nodes[index]['contents']
        is_code_node = isCodeNode(data, index)
        if is_code_node:
            data = self._dedent(data)
            data = data[8:]
            self.nodes[index]['contents'] = data
        index += 1


def _contents2String(self):
    """nodes[index]['contents'] is a list of string,
    all the processing required is line oriented.
    Leo requires a string for bodyString.

    index = 0
    num_nodes = len(self.nodes)
    while index < num_nodes:
        list = self.nodes[index]['contents']
        string = '\n'.join(list)
        self.nodes[index]['contents'] = string
        index += 1


def processNodes(self):
    """Step through the list of nodes created by
    parseReST creating the appropriate Leo nodes


    # Create root node as a sibling of current node
    # Creating a new node provides crude versioning,
    # and avoids issues of replacing the existing tree with
    # the one being downloaded
    root = self.p.insertAfter()
    self.levels['root'] = root

    rootstring = self.nodes[0]['contents']
    roottitle = self._setRootNodeHeadString(rootstring)
    if not roottitle:
        roottitle = self.nodes[0]['title']

    # step through the rest of the nodes
    index = 1
    while index < len(self.nodes):
        uc = self.nodes[index]['underline_character']
        title = self.nodes[index]['title']
        contents = self.nodes[index]['contents']

        # this level exists, insert the node
        if self.levels.has_key(uc):
            # get parent of this node
            parent_index = self.underline_characters.index(uc) - 1
            parent_uc = self.underline_characters[parent_index]
            current = self.levels[parent_uc].insertAsLastChild()
            self.levels[uc] = current

        # if this is the first time this uc is encountered
        # it means we are creating a new sublevel
        # create the level then insert the node
            # if we are descending to a new level, the parent
            # underline character is currently the last one
            parent = self.levels[self.underline_characters[-1] ]
            current = parent.insertAsLastChild()
            self.levels[uc]  = current

        index += 1

class ZWiki

class ZWiki:
    <<class ZWiki declarations>>

<<class ZWiki declarations>>

from zope.testbrowser.browser import Browser


def __init__ (self, url, auth = None, cookie = None, create = False):
    """ZWiki init

    # create a Browser instance
    self.auth = auth
    self.cookie = cookie
    self.create = create
    self.url = url
    self.browser = self.Browser()


def _addHeaders(self):

    if self.auth:
        self.browser.addHeader('Authorization', 'Basic %s' % self.auth)
    if self.cookie:
        self.browser.addHeader('Cookie', self.cookie)


def _string2DateTime(self, s):
    """Convert the string used by ZWiki ThePage/lastEditTime
    to a datetime.datetime instance

    import datetime, pytz

    date, time, tzone = s.split()
    date = date.split('/')
    time = time.split(':')
    tz = pytz.timezone(tzone)

    web_tzone = pytz.timezone(tzone)
    dt = datetime.datetime(int(date[0]), int(date[1]), int(date[2]),
                            int(time[0]), int(time[1]), int(time[2]),
                            tzinfo = tz)
    return dt


def _parseURL(self):
    """Get self.page and self.base from the url,
    assemble url's for editform, page create and lastEditTime
    url = self.url
    self.page = url[url.rfind('/')+1:]
    self.base = url[0:url.rfind('/')]
    self.url_edit ='%s/%s/editform' % (self.base,self.page)
    self.url_create = '%s/FrontPage/editform?page=%s' % (self.base, self.page)
    self.url_lastEditTime = '%s/lastEditTime' % url
    self.url_manage = '%s/manage' % url
    self.url_createFile = '%s/manage_addProduct/OFSP/fileAdd' % self.base


def _createFile(self):

    form_index = 0
    id_name = 'id'
    submit_name = 'submit'

    form0 = self.browser.getForm(index = form_index)
    id_field = form0.getControl(name = id_name)
    submit_field = form0.getControl(name = submit_name)

    id_field.value = self.page


def getLastEditTime(self):
    """Return the contents of page/lastEditTime

    return self.browser.contents


def putReST(self, data):

    # TODO figure out why
    # except HTPPError:
    # results in
    # # NameError: name 'HTTTPError' is not defined
    except :
        if self.create:
            raise "The page does not exist, and 'create' is not configured"

    form_index = 0
    textarea_name = 'text'
    save_name = "edit:method"

    form = self.browser.getForm(index = form_index)
    textarea = form.getControl(name=textarea_name)
    save = form.getControl(name=save_name)
    if type(data) != type("string"):
        data = '\n'.join(data)
    textarea.value = data


def getReST(self):
    Get the contents of the text form on ThePage/editform

    return contents

    response = self.browser.open(self.url_edit)

    form_index = 0
    textarea_name = 'text'
    save_name = "edit:method"

    form = self.browser.getForm(index = form_index)
    textarea = form.getControl(name=textarea_name)
    self.contents = textarea.value
    return self.contents


def putFile(self, data):
    """Upload a file to a ZWiki.

    Especially css.
    This version is pasting into a textarea, so it's only good for text files
    if not self.auth:
        raise "you must have manage authentication to use putFile()"
        if not self.create:
            raise "You must set @create_page = True"

    form_index = 0
    textarea_name = 'filedata:text'
    save_name = 'manage_edit:method'

    form = self.browser.getForm(index = form_index)
    textarea = form.getControl(name = textarea_name)
    save = form.getControl(name = save_name)

    if type(data) != type("string"):
        data = '\n'.join(data)

    textarea.value = data


def getFile(self):

    if not self.auth:
        raise "you must have manage authentication to use putFile()"

    form_index = 0
    textarea_name = 'filedata:text'
    save_name = 'manage_edit:method'


    form = self.browser.getForm(index = form_index)
    textarea = form.getControl(name = textarea_name)
    save = form.getControl(name = save_name)

    data = textarea.value
    return data

class Button

class Button:
    """Wrapper for rst2leo 'upload' and 'download' buttons


def __init__(self, position, commander, leoGlobals):

    self.p = position
    self.c = commander
    self.g = leoGlobals

    self.body = self.p.bodyString().split('\n')
    self.head = self.p.headString()

    self.url = getConfig(self.body, '.. url:')
    auth_key = getConfig(self.body, '.. auth:')
    self.auth = self.c.config.getString(auth_key)
    cookie_key = getConfig(self.body, '.. cookie:')
    self.cookie = self.c.config.getString(cookie_key)
    self.create = self.c.config.getBool('create_page')


def _stripCodeMarkers(self, data):
    """@data is a string containing the ReST version of the Leo node
    convert to list for processing to remove the **code** markers
    added by the rst3 plugin, return as a string.

    data = data.split('\n')
    fixed_data = []
    for line in data:
        if line.strip() != '**code**:':

    return '\n'.join(fixed_data)


def upload(self):
    p = self.p
    c = self.c
    g = self.g

    # uploading type = ReST
    if p.headString().startswith('@rst'):
        g.es("Uploading ReST")
        # activate rst3 plugin, get ReST version of this node tree
        import leoPlugins
        rst3 = leoPlugins.getPluginModule('rst3')
        rst3.SilverCity = None
        controller = rst3.controllers.get(c)
        for p in p.self_and_parents_iter():
            if p.headString().startswith('@rst '):
        data = controller.source

        data = self._stripCodeMarkers(data)

        z = ZWiki(self.url, self.auth, self.cookie, self.create)
        g.es('Upload of %s successful' % self.url)

    # css files have the URL in the headline
    elif self.head.lower().find('.css') > -1:
        g.es("Uploading file type")
        url = self.head.strip()
        z = ZWiki(url, self.auth, self.cookie, self.create)
        g.es('Upload of %s successful' % self.url)

        g.es('rst2leo currently only supports ReST and .css files')


def download(self):
    p = self.p
    c = self.c
    g = self.g

    if p.headString().startswith('@rst'):
#        from code.rst2leo import ParseReST, BuildLeo
#        url = getConfig(body, '.. url:')
        z = ZWiki(self.url, self.auth, self.cookie)
        data = z.getReST()
        parsed = ParseReST(data)
        sections = parsed.processLines()
        nodes = BuildLeo(sections, p, self.g)
        g.es('Download of %s successful' % self.url)

    elif self.head.lower().find('.css') > -1:
        self.url = self.head.strip()
        z = ZWiki(self.url, self.auth, self.cookie)
        data = z.getFile()
        g.es('Download of %s successful' % self.url)
        g.es('rst2leo currently only supports ReST and .css files')


def getConfig(text, tag):
    """Utility method to retrieve config data

    @text is a string or list of strings
    @tag and it's value must begin, and be alone on, a line.

    # if string or unicode,  convert to list
    if type(text) == type("")\
        or type(text) == type(u""):

        text = text.split("\n")

    for line in text:
        if line.find(tag) > -1:
            start = len(tag)
            return line[start:].strip()