Top

ccg_nlpy.download module

import codecs
import configparser
import logging
import os
import platform
import subprocess
import six

logger = logging.getLogger(__name__)

MODEL_FOLDER = "model_{}"
CONFIG_FILENAME = "config.cfg"
DEFAULT_CONFIG_ROOT_DIRECTORY = "~{0}.ccg_nlpy{0}".format(os.path.sep)
DEFAULT_CONFIG_VERSION = "4.0.9"
MAVEN_COMMAND = "mvn dependency:copy-dependencies -DoutputDirectory={} -Dsilent=True"
POM_TEMPLATE = """

    4.0.0

    org.cogcomp
    ccg_nlpy-pipeline-dependencies
    1.0-SNAPSHOT

    
        
            CogcompSoftware
            CogcompSoftware
            http://cogcomp.org/m2repo/
        
    

    
        
            edu.illinois.cs.cogcomp
            illinois-nlp-pipeline
            ##VERSION##
        
    


"""


def _shell_argument():
    """Identify if the current platform is a Windows installation."""
    return 'windows' in platform.system().lower()


def _parse_default_config(root_directory, args=None):
    """Parse default configuration for model_download"""

    default_config_file = os.path.join(root_directory, CONFIG_FILENAME)
    package_config_file = os.path.dirname(os.path.realpath(__file__)) + '/config/pipeline.cfg'

    config = configparser.ConfigParser()
    if os.path.exists(default_config_file):
        with codecs.open(default_config_file, mode='r', encoding='utf-8') as f:
            config.read_string(f.read())
    else:
        with codecs.open(package_config_file,mode='r',encoding='utf-8') as f:
            config.read_string(f.read())

    if 'model_download' not in config:
        config['model_download'] = {}

    if args is not None and args.version is not None:
        version = args.version
    elif 'version' not in config['model_download']:
        version = DEFAULT_CONFIG_VERSION
    else:
        version = config['model_download']['version']

    config['model_download']['version'] = version

    return (config, default_config_file)


def _check_maven_installed():
    """Method to check if maven is installed and on the classpath"""

    try:
        output = subprocess.check_output(
            ["mvn", "--version"], shell=_shell_argument())
        logger.debug(output)
    except Exception:
        logger.error(
            'Maven installation not found!\n\
            Please install Apache Maven and add it to the classpath.',
            exc_info=True)
        raise


def _create_or_update_pom_file(file_path, version):
    """
    Create the pom.xml file needed for maven to download dependencies.
    """

    logger.info("Creating pom file")

    if six.PY2:
        # Python 2 - Write as a binary file and replace newline on windows
        # to universal newline.
        with open(file_path, 'wb') as file:
            pom_data = POM_TEMPLATE.replace("##VERSION##", version)
            pom_data = pom_data.replace('\r\n', '\n')
            file.write(pom_data)
    else:
        # Python 3 - use newline as \n always. Maven does not work otherwise.
        with open(file_path, "w", newline='\n') as file:
            pom_data = POM_TEMPLATE.replace("##VERSION##", version)
            file.write(pom_data)


def _download_jars(model_directory, config_directory, version):
    """
    Download Jars into the model_directory parameter

    Note: This method invokes maven internally to download dependencies.
    """

    if not os.path.exists(model_directory):
        os.makedirs(model_directory)

    logger.info('Root Directory = {}'.format(config_directory))
    logger.info('Model Directory = {}'.format(model_directory))

    try:
        command = MAVEN_COMMAND.format(model_directory)
        logger.debug(command)

        command_parts = command.split()
        proc = subprocess.Popen(
            command_parts,
            cwd=config_directory,
            stdout=subprocess.PIPE,
            shell=_shell_argument())

        # Calling proc.communicate() waits for the subprocess to complete.
        output, _ = proc.communicate()
        logger.info(output)
    except Exception:
        logger.error('Error while downloading jar files.', exc_info=True)
        raise

def recover_model_config():
    """
    Function to recover the model config if user accidentally deleted it
    """
    try:
        root_directory = get_root_directory()
        default_config, config_file_path = _parse_default_config(root_directory)
    
        with codecs.open(config_file_path, mode='w', encoding='utf-8') as file:
            default_config.write(file)
    except:
        logger.error("Error while recovering model config.", exc_info=True)


def get_root_directory():
    """Get the root config/model directory"""

    root_directory = os.path.expanduser(DEFAULT_CONFIG_ROOT_DIRECTORY)

    if not os.path.exists(root_directory):
        os.makedirs(root_directory)

    return root_directory


def get_model_path():
    """Returns the path of the JAR models."""

    root_directory = get_root_directory()
    default_config, _ = _parse_default_config(root_directory)
    version = default_config['model_download']['version']

    return os.path.join(root_directory, MODEL_FOLDER.format(version))


def main(args):
    """Default handler function"""

    logger.info("Starting download")
    try:
        _check_maven_installed()

        # Get default config along with args override.
        root_directory = get_root_directory()
        (config, config_file_path) = _parse_default_config(
            root_directory, args)
        version = config['model_download']['version']

        # Create/Update POM file.
        pom_file_path = os.path.join(root_directory, "pom.xml")
        logger.info(pom_file_path)
        _create_or_update_pom_file(pom_file_path, version)

        # Download model jars according to the version specified.
        jar_directory = os.path.join(root_directory,
                                     MODEL_FOLDER.format(version))
        _download_jars(jar_directory, root_directory, version)

        # Write the updated config file if download was successful.
        with codecs.open(config_file_path, mode='w', encoding='utf-8') as file:
            config.write(file)

        logger.info("Model download successful.")
    except Exception:
        logger.exception("Download failed.", exc_info=True)

Module variables

var CONFIG_FILENAME

var DEFAULT_CONFIG_ROOT_DIRECTORY

var DEFAULT_CONFIG_VERSION

var MAVEN_COMMAND

var MODEL_FOLDER

var POM_TEMPLATE

var logger

Functions

def get_model_path(

)

Returns the path of the JAR models.

def get_model_path():
    """Returns the path of the JAR models."""

    root_directory = get_root_directory()
    default_config, _ = _parse_default_config(root_directory)
    version = default_config['model_download']['version']

    return os.path.join(root_directory, MODEL_FOLDER.format(version))

def get_root_directory(

)

Get the root config/model directory

def get_root_directory():
    """Get the root config/model directory"""

    root_directory = os.path.expanduser(DEFAULT_CONFIG_ROOT_DIRECTORY)

    if not os.path.exists(root_directory):
        os.makedirs(root_directory)

    return root_directory

def main(

args)

Default handler function

def main(args):
    """Default handler function"""

    logger.info("Starting download")
    try:
        _check_maven_installed()

        # Get default config along with args override.
        root_directory = get_root_directory()
        (config, config_file_path) = _parse_default_config(
            root_directory, args)
        version = config['model_download']['version']

        # Create/Update POM file.
        pom_file_path = os.path.join(root_directory, "pom.xml")
        logger.info(pom_file_path)
        _create_or_update_pom_file(pom_file_path, version)

        # Download model jars according to the version specified.
        jar_directory = os.path.join(root_directory,
                                     MODEL_FOLDER.format(version))
        _download_jars(jar_directory, root_directory, version)

        # Write the updated config file if download was successful.
        with codecs.open(config_file_path, mode='w', encoding='utf-8') as file:
            config.write(file)

        logger.info("Model download successful.")
    except Exception:
        logger.exception("Download failed.", exc_info=True)

def recover_model_config(

)

Function to recover the model config if user accidentally deleted it

def recover_model_config():
    """
    Function to recover the model config if user accidentally deleted it
    """
    try:
        root_directory = get_root_directory()
        default_config, config_file_path = _parse_default_config(root_directory)
    
        with codecs.open(config_file_path, mode='w', encoding='utf-8') as file:
            default_config.write(file)
    except:
        logger.error("Error while recovering model config.", exc_info=True)