#!/usr/bin/env python
##############################################################################
# Copyright (c) CERN, 2012.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at #
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS
# OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##############################################################################

"""Ginfo - Developped by Ivan Calvet for CERN - ivan.calvet@cern.ch"""

import ldap
import sys
import getopt
try:
	import json
except:
	import simplejson as json
import os
import re
import urllib2
import signal

VERSION = '0.2.6'

# configuration dictionnary that will be filled with the submitted options
CONF = {}

# long_option: [short_option, parameter_needed]
OPTIONS = {
    'host': [None, True],
    'registry': ["r", True],
    'bind': ["b", True],
    'list': ["l", True],
    'clean': [None, False],
    'strict': ["s", False],
    'csv': ["c", False],
    'json': ["j", False],
    'emi': ["e", False],
    'timeout': [None, True],
    'verbose': ["v", False],
    'version': ["V", False],
    'help': ["h", False],
    'cap': [None, True],
    'domain': ["d", True],
    'endid': [None, True],
    'id': ["i", True],
    'imp': ["m", True],
    'impv': [None, True],
    'int': ["n", True],
    'intv': [None, True],
    'ql': ["q", True],
    'tech': [None, True],
    'type': ["t", True],
    'url': ["u", True],
    'vo': [None, True],
    }

# ldap request attribute: [prefix_object, object_class, key, output_list, EMI_name]
ATTRIBUTES = {
    'cap': ['', 'Endpoint', 'Capability', True, 'Service_Endpoint_Capability'],
    'domain': ['', 'Service', 'AdminDomainForeignKey', False, 'Service_Admin_Domain'],
    'endid': ['', 'Endpoint', 'ID', False, 'Service_Endpoint_ID'],
    'id': ['', 'Service', 'ID', False, 'Service_ID'],
    'imp': ['', 'Endpoint', 'ImplementationName', False, 'Service_Endpoint_Implementation_Name'],
    'impv': ['', 'Endpoint', 'ImplementationVersion', False, 'Service_Endpoint_Implementation_Version'],
    'int': ['', 'Endpoint', 'InterfaceName', False, 'Service_Endpoint_Interface_Name'],
    'intv': ['', 'Endpoint', 'InterfaceVersion', False, 'Service_Endpoint_Interface_Version'],
    'ql': ['', 'Endpoint', 'QualityLevel', False, 'Service_Endpoint_Quality_Level'],
    'tech': ['', 'Endpoint', 'Technology', False, 'Service_Endpoint_Technology'],
    'type': ['', 'Service', 'Type', False, 'Service_Type'],
    'url': ['', 'Endpoint', 'URL', False, 'Service_Endpoint_URL'],
    'vo': ['Access', 'Policy', 'Rule', True, None],
    }

# keys that link ldap objects to Endpoint object [objectKey, EndpointKey]
OBJECT_KEY = {'Service': ['ID', 'EndpointServiceForeignKey'], 'AccessPolicy': ['EndpointForeignKey', 'EndpointID']}

TIMEOUT = 15

if sys.version_info[:2] < (2,5):
    def all(iterable):
        for element in iterable:
            if not element:
                return False
        return True

def main(argv):
    """Main function that launches the other functions""" 

    parse_options(argv)
    validate_conf()
    if 'list' in CONF and 'registry' in CONF:
        result = list_attr_registry() # option --list and --registry  
    elif 'list' in CONF:
        result = list_attr() # option --list
    elif 'registry' in CONF:
        result = list_services_registry() # option --registry
    else:
        result = list_services() # get all the informations about services
    if 'verbose' in CONF:
        print ''
    result = clean(result)
    print serialize_output(result)
    sys.exit()

def parse_options(argv):
    """Parse the selected options and put them in a config dictionnary""" 

    # parse the options
    short_options = ''
    long_options = []
    long_options_bis = [] # options without any short option
    for i in OPTIONS:
        if OPTIONS[i][0]:
            short_options += OPTIONS[i][0]
            long_options.append(i)
            if OPTIONS[i][1]:
                short_options += ':'
                long_options[-1] += '='
        else:
            long_options_bis.append(i)
            if OPTIONS[i][1]:
                long_options_bis[-1] += '='
    long_options.extend(long_options_bis)
    try:
        opts, args = getopt.getopt(argv, short_options, long_options)
    except getopt.error, err:
        if err.opt in ("l", "list"):
            sys.exit(list_usage())
        else:
            sys.exit(usage())

    # put the options in the config dictionnary
    for opt, arg in opts:
        opt = opt[opt.rfind('-')+1:]
        for i in OPTIONS:
            if opt in (i, OPTIONS[i][0]):
                if i not in CONF:
                    if OPTIONS[i][1]:
                        CONF[i] = arg
                    else:
                        CONF[i] = True
                    break 
                else:
                    sys.exit('Error: use options not more than once.')

    # put the arguments in the config dictionnary
    CONF['args'] = args
    if len(args) == 0:
        CONF['args'] = ATTRIBUTES.keys() # all by default
        CONF['args'].sort()

def validate_conf():
    """Prints verbose messages and checks for errors""" 
        
    # options
    if 'help' in CONF:
        print usage()
        sys.exit()
    if 'version' in CONF:
        print os.path.basename(sys.argv[0]) +' V'+VERSION
        sys.exit()
    if 'verbose' in CONF:
        print 'Verbose mode enabled'
    if all(i in CONF for i in ['csv', 'json', 'emi']) and 'list' not in CONF:
        sys.exit('Error: choose between csv, json and emi.')
    elif 'csv' in CONF and 'json' in CONF and 'list' not in CONF:
        sys.exit('Error: choose between csv and json.')
    elif 'csv' in CONF and 'emi' in CONF and 'list' not in CONF:
        sys.exit('Error: choose between csv and emi.')
    elif 'emi' in CONF and 'json' in CONF and 'list' not in CONF:
        sys.exit('Error: choose between json and emi.')
    elif 'csv' in CONF and 'list' not in CONF:
        if 'verbose' in CONF:
            print 'Output in csv formating'
    elif 'json' in CONF and 'list' not in CONF:
        if 'verbose' in CONF:
            print 'Output in json formating'
    elif 'emi' in CONF and 'list' not in CONF:
        for k in ATTRIBUTES:
            if not ATTRIBUTES[k][4] and k in CONF['args']:
                CONF['args'].remove(k)
        if 'verbose' in CONF:
            print 'Output in emi formating'
    if 'host' in CONF and 'registry' in CONF:
        sys.exit('Error: choose between host and registry option.')
    elif 'registry' in CONF:
        if 'verbose' in CONF:
            print 'The following EMI registry will be used:', CONF['registry']
    elif 'host' in CONF:
        if 'verbose' in CONF:
            print 'The following host will be used:', CONF['host']
    else:
        if 'LCG_GFAL_INFOSYS' in os.environ:
            CONF['host'] = os.environ['LCG_GFAL_INFOSYS']
            if 'verbose' in CONF:
                print 'The following host will be used:', CONF['host']
        else:
            sys.exit(usage())
    if 'bind' in CONF:
        if 'verbose' in CONF:
            print 'The following binding will be used:', CONF['bind']
    else:
        CONF['bind'] = 'o=glue'
        if 'verbose' in CONF:
            print 'The default binding will be used:', CONF['bind']
    if 'list' in CONF:
        if CONF['list'] not in ATTRIBUTES.keys():
            for i in ATTRIBUTES:
                if ATTRIBUTES[i][1]+ATTRIBUTES[i][2] == CONF['list']:
                    CONF['list'] = i
                    break
            if CONF['list'] not in ATTRIBUTES.keys():
                sys.exit('Error: '+CONF['list']+' wrong attribute.')
        if 'verbose' in CONF:
            print 'List all the possible values for the following attribute:', ATTRIBUTES[CONF['list']][1]+ATTRIBUTES[CONF['list']][2]
            for i in ['csv', 'json', 'emi'] + ATTRIBUTES.keys():
                if i in CONF:
                    print 'Option --'+i+' is not compatible with --list option.'
    if 'clean' in CONF and 'strict' in CONF:
        sys.exit('Error: choose between clean and strict.')
    elif 'clean' in CONF:
        if 'verbose' in CONF:
            print 'Results have been cleaned.'
    elif 'strict' in CONF:
        if 'verbose' in CONF:
            print 'Results have been cleaned strictly.'
    if 'timeout' in CONF:
        if 'verbose' in CONF:
            print 'Ldap timeout has been set to '+CONF['timeout']+' second(s).'

    # attributes
    for i in ATTRIBUTES.keys():
        if i in CONF:
            if 'verbose' in CONF and 'list' not in CONF:
                print 'Filter services by the following '+ATTRIBUTES[i][1]+ATTRIBUTES[i][2]+':', CONF[i]
    
    # arguments
    if 'args' in CONF and 'list' not in CONF:
        display_list = []
        arg_list = []
        for arg in CONF['args']:
            if arg not in ATTRIBUTES.keys():
                wrong_arg = True
                for i in ATTRIBUTES:
                    if ATTRIBUTES[i][1]+ATTRIBUTES[i][2] == arg:
                        if i not in arg_list:
                            arg_list.append(i)
                            display_list.append(arg)
                        wrong_arg = False
                        break
                if wrong_arg:
                    sys.exit('Error: '+arg+' is a wrong attribute.')
            elif arg not in arg_list:
                arg_list.append(arg)
                display_list.append(ATTRIBUTES[arg][1]+ATTRIBUTES[arg][2])
        CONF['args'] = arg_list
        if 'verbose' in CONF:
            print 'The following attribute(s) will be displayed:', ' '.join(display_list)

def usage():
    """Returns the usage message""" 
    
    return '''Usage: ginfo [options] [attributes]

    List URLs of services along with other optional attributes.

        --host      host        Specify a host to query. By default the
                                environmental variable LCG_GFAL_INFOSYS will be
                                used.
    -r, --registry  registry    Specify an EMI registry to query.
    -b, --bind      binding     Specify the binding (o=glue by default).
    -l, --list      attribute   List all the possible values of the specified
                                attribute.
        --clean                 Clean results in replacing all invalid data.
    -s, --strict                Clean strictly results in replacing all invalid
                                data.
    -c, --csv                   Output in CSV format
    -j, --json                  Output in JSON format
        --timeout               Change the ldap timeout (15 seconds by default).
    -v, --verbose               Enable verbose mode
    -V, --version               Prints the version of ginfo
    -h, --help                  Prints this helpful message

    Addition options to filter services by the specified attribute:

        --cap       EndpointCapability
    -d, --domain    ServiceAdminDomainForeignKey
        --endid     EndpointID
    -i, --id        ServiceID
    -m, --imp       EndpointImplementationName
        --impv      EndpointImplementationVersion
    -n, --int       EndpointInterfaceName
        --intv      EndpointInterfaceVersion
    -q, --ql        EndpointQualityLevel
	--tech      EndpointTechnology
    -t, --type      ServiceType
    -u, --url       EndpointURL
        --vo        PolicyRule

    Available attributes to display are:

        cap       EndpointCapability
        domain    ServiceAdminDomainForeignKey
        endid     EndpointID
        id        ServiceID
        imp       EndpointImplementationName
        impv      EndpointImplementationVersion
        int       EndpointInterfaceName
        intv      EndpointInterfaceVersion
        ql        EndpointQualityLevel
        tech      EndpointTechnology
        type      ServiceType
        url       EndpointURL
        vo        PolicyRule'''

def list_usage():
    """Returns the usage message for list option"""

    return '''Available attributes are:

        cap       EndpointCapability
        domain    ServiceAdminDomainForeignKey
        endid     EndpointID
        id        ServiceID
        imp       EndpointImplementationName
        impv      EndpointImplementationVersion
        int       EndpointInterfaceName
        intv      EndpointInterfaceVersion
        ql        EndpointQualityLevel
        tech      EndpointTechnology
        type      ServiceType
        url       EndpointURL
        vo        PolicyRule'''

def request(filter=None):
    """Returns the result of the ldap request with the filter given"""
    try:
        t = int(CONF['timeout'])
    except (ValueError, KeyError):
        t = TIMEOUT
    signal.alarm(t)
    if 'host' in CONF:
        try:
            port = ''
            if ':' not in CONF['host']:
                port = ':2170'
            con = ldap.initialize('ldap://'+CONF['host']+port)
            if filter:
                result = con.result(con.search(CONF['bind'], ldap.SCOPE_SUBTREE, filter))[1]
            else:
                result = con.result(con.search(CONF['bind'], ldap.SCOPE_SUBTREE))[1]
        except ldap.SERVER_DOWN:
            sys.exit('Error: Can\'t contact the LDAP server. Please check your host.')
    elif 'registry' in CONF:
        url = CONF['registry']
        if filter:
            separator = '&'
            if '?' not in url:
                separator = '?'
            url += separator + filter
        try:
            req = urllib2.Request(url)
        except ValueError:
            sys.exit('Error: Can\'t contact the url.')
        handle = urllib2.urlopen(req)
        result = json.loads(handle.read())
    return result

def list_services_registry():
    """Returns a dictionary of filtered results from an http request in the EMI registry"""

    filter = ''
    for i in ATTRIBUTES:
        if ATTRIBUTES[i][4]:
            if i in CONF:
                if filter:
                    filter += '&'
                filter += ATTRIBUTES[i][4] + '=' + CONF[i]
    for j in CONF:
        if j in ATTRIBUTES  and not ATTRIBUTES[j][4]:
            if 'verbose' in CONF:
                sys.exit('Error: No result. Verify search parameters.')
            else:
                return {}
    list_results = request(filter)
    dic = {}
    for res in list_results:
        _id = res['_id']['$oid']
        # filter by id
        if 'id' in CONF and CONF['id'] != _id:
            continue
        dic[_id] = {}
        for i in ATTRIBUTES:
            att = ATTRIBUTES[i][4]
            if att:
                value = None
                if att == '_id':
                    value = _id
                elif att in res:
                    value = res[att]
                dic[_id][i] = value
        for j in CONF['args']:
            if j not in dic[_id]:
                dic[_id][j] = None
    if len(dic) == 0 and 'verbose' in CONF:
        sys.exit('Error: No result. Verify search parameters.')
    return dic

def list_services():
    """Returns a dictionary of filtered results from a ldap request""" 

    ldap_filter = {'Endpoint': ''}
    result = {}
    selection_list = {}

    # Generate the filters 
    for i in ATTRIBUTES:
        at1 = ATTRIBUTES[i][0]+ATTRIBUTES[i][1]
        at2 = ATTRIBUTES[i][1]+ATTRIBUTES[i][2]
        if i in CONF:
            if at1 not in ldap_filter:
                ldap_filter[at1] = ''
            tmp_filter = '(GLUE2'+at2+'='+CONF[i]+')'
            if at2 == 'PolicyRule':
                tmp_filter = '(|(GLUE2PolicyRule=ALL)'+tmp_filter+')'
            ldap_filter[at1] += tmp_filter
    # Execute the filtered ldap requests and put results in a list
    for i in OBJECT_KEY.keys():
        if i in ldap_filter:
            result[i] = request('(&(objectClass=GLUE2'+i+')'+ldap_filter[i]+')')
            selection_list[i] = []
            for res in result[i]:
                selection_list[i].append(res[1]['GLUE2'+i+OBJECT_KEY[i][0]][0])
    
    # Main loop using Endpoint object
    result['Endpoint'] = request('(&(objectClass=GLUE2Endpoint)'+ldap_filter['Endpoint']+')')
    dic = {} # Final results
    dic2 = {} # Tmp results
    multiples = []
    for res in result['Endpoint']: # for each endpoint entry
        ids = {}
        for i in OBJECT_KEY:
            ids[i] = res[1]['GLUE2'+OBJECT_KEY[i][1]][0]
        # Executes the filtering by other objects
        stop = False
        for i in OBJECT_KEY.keys():
            if i in selection_list and ids[i] not in selection_list[i]:
                stop = True
                break
        if stop:
            continue
        # Fill in the dictionnary of one entry (key = ServiceID)
        if ids['Service'] in dic and ids['Service'] not in multiples:
            multiples.append(ids['Service'])
        dic[ids['Service']] = {}
        for attr in CONF['args']: # for all displayed attributes
            at1 = ATTRIBUTES[attr][0]+ATTRIBUTES[attr][1]
            at2 = ATTRIBUTES[attr][1]+ATTRIBUTES[attr][2]
            dic[ids['Service']][attr] = None # initialization
            if ATTRIBUTES[attr][3]:
                dic[ids['Service']][attr] = [] # initialization for lists
            if attr in CONF and not ATTRIBUTES[attr][3] and CONF[attr].find('*') == -1: # if the attribute is filtered and not a list
                dic[ids['Service']][attr] = CONF[attr]
            else:
                if at1 == 'Endpoint':
                    if 'GLUE2'+at2 in res[1]:
                        dic[ids['Service']][attr] = res[1]['GLUE2'+at2] # get the value
                        if not ATTRIBUTES[attr][3]: # if attr not a list, take the first element
                            dic[ids['Service']][attr] = dic[ids['Service']][attr][0]
                else: # Service and AccessPolicy objects
                    if at1 not in result:
                        result[at1] = request('(objectClass=GLUE2'+at1+')')
                    if attr not in dic2: # temporary dictionnary to store results of other objects
                        dic2[attr] = {}
                        for res2 in result[at1]:
                            if 'GLUE2'+at2 in res2[1]:
                                dic2[attr][res2[1]['GLUE2'+at1+OBJECT_KEY[at1][0]][0]] = res2[1]['GLUE2'+at2]
                            else:
                                dic2[attr][res2[1]['GLUE2'+at1+OBJECT_KEY[at1][0]][0]] = None
                    if ids[at1] in dic2[attr]:
                        dic[ids['Service']][attr] = dic2[attr][ids[at1]] # get the value
                        if not ATTRIBUTES[attr][3] and dic[ids['Service']][attr]: # if attr not a list, take the first element
                            dic[ids['Service']][attr] = dic[ids['Service']][attr][0]
    if multiples and 'verbose' in CONF:
        print 'Warning: these service ids were found many times and only the last entry will be displayed:\n'+'\n'.join(multiples)
    if len(dic) == 0 and 'verbose' in CONF:
        sys.exit('Error: No result. Verify search parameters.')
    return dic

def list_attr_registry():
    """Returns a list of values in the EMI registry for a given attribute"""

    list_results = request()
    attr_list = []
    att = ATTRIBUTES[CONF['list']][4]
    if not att:
        sys.exit("Error: "+CONF['list']+" canno't be found in the EMI registry.")
    for res in list_results:
        if att in res and res[att]:
            if ATTRIBUTES[CONF['list']][3]:
                for j in res[att]:
                    if j not in attr_list:
                        attr_list.append(j)
            elif res[att] not in attr_list:
                if att == '_id':
                    attr_list.append(res[att]['$oid'])
                else:
                    attr_list.append(res[att])
        elif 'None' not in attr_list:
            attr_list.append('None')
    return attr_list

def list_attr():
    """Returns a list of values for a given attribute""" 
    
    object_class = ATTRIBUTES[CONF['list']][0]+ATTRIBUTES[CONF['list']][1]
    key = ATTRIBUTES[CONF['list']][1]+ATTRIBUTES[CONF['list']][2]
    result = request('objectClass=GLUE2'+object_class)
    attr_list = []
    for res in result:
        if 'GLUE2'+key in res[1]:
            if ATTRIBUTES[CONF['list']][3]:
                for i in res[1]['GLUE2'+key]:
                    if i not in attr_list:
                        attr_list.append(i)
            if res[1]['GLUE2'+key][0] not in attr_list:
                attr_list.append(res[1]['GLUE2'+key][0])
        else:
            if 'None' not in attr_list:
                attr_list.append('None')
    attr_list.sort()
    return attr_list

def clean(result):
    """Clean all the results"""
    values = {
        'ql': [
            'development',
            'pre-production',
            'production',
            'testing'
            ],
        'cap': [
            'data.access.flatfiles',
            'data.access.relational',
            'data.access.xml',
            'data.management.replica',
            'data.management.storage',
            'data.management.transfer',
            'data.naming.resolver',
            'data.naming.scheme',
            'data.transfer',
            'executionmanagement.candidatesetgenerator',
            'executionmanagement.dynamicvmdeploy',
            'executionmanagement.executionandplanning',
            'executionmanagement.jobdescription',
            'executionmanagement.jobexecution',
            'executionmanagement.jobmanager',
            'executionmanagement.reservation',
            'information.discovery',
            'information.logging',
            'information.model',
            'information.monitoring',
            'information.provenance',
            'security.accounting',
            'security.attributeauthority',
            'security.authentication',
            'security.authorization',
            'security.credentialstorage',
            'security.delegation',
            'security.identymapping'
        ],
        'int': [
           'MyProxy',
           'SRM',
           'VOBOX',
           'bdii_site',
           'bdii_top',
           'dcap',
           'emi.storm',
           'gsidcap',
           'gsiftp',
           'http',
           'https',
           'lcg-file-catalog',
           'lcg-local-file-catalog',
           'org.glite.ChannelManagement',
           'org.glite.Delegation',
           'org.glite.FileTransfer',
           'org.glite.RTEPublisher',
           'org.glite.ce.ApplicationPublisher',
           'org.glite.ce.CREAM',
           'org.glite.ce.Monitor',
           'org.glite.lb.Server',
           'org.glite.voms',
           'org.glite.voms-admin',
           'org.glite.wms.WMProxy',
           'org.globus.gram',
           'org.nordugrid.gridftpjob',
           'org.nordugrid.ldapglue1',
           'org.nordugrid.ldapglue2',
           'org.nordugrid.ldapng',
           'xroot'
        ],
        'type': [
            'org.dcache.storage',
            'org.glite.wms.WMProxy',
            'org.glite.lb.Server',
            'org.glite.ce.ApplicationPublisher',
            'org.glite.ce.CREAM',
            'org.glite.ce.Monitor',
            'org.glite.rgma.Consumer',
            'org.glite.rgma.Registry',
            'org.glite.rgma.Schema',
            'org.glite.rgma.Browser',
            'org.glite.rgma.PrimaryProducer',
            'org.glite.rgma.SecondaryProducer',
            'org.glite.rgma.OnDemandProducer',
            'org.glite.RTEPublisher',
            'org.glite.voms',
            'org.glite.voms-admin',
            'org.glite.AMGA',
            'org.glite.ChannelManagement',
            'org.glite.FileTransfer',
            'org.glite.FileTransferStats',
            'org.glite.ChannelAgent',
            'org.glite.Delegation',
            'org.glite.KeyStore',
            'org.nordugrid.execution.arex',
            'SRM',
            'gsiftp',
            'org.edg.gatekeeper',
            'it.infn.GridICE',
            'MyProxy',
            'GUMS',
            'gridmap-file',
            'GridCat',
            'edu.caltech.cacr.monalisa',
            'OpenSSH',
            'xroot',
            'BDII',
            'bdii_site',
            'bdii_top',
            'VOBOX',
            'msg.broker.rest',
            'msg.broker.stomp',
            'msg.broker.stomp-ssl',
            'msg.broker.openwire',
            'msg.broker.openwire-ssl',
            'org.lcg.Frontier',
            'org.lcg.Squid',
            'Nagios',
            'National-NAGIOS',
            'Project-NAGIOS',
            'Regional-NAGIOS',
            'data-location-interface',
            'local-data-location-interface',
            'lcg-file-catalog',
            'lcg-local-file-catalog',
            'pbs.torque.server',
            'pbs.torque.maui',
            'other'
        ],
    }
    
    if 'list' in CONF and 'vo' in CONF['list']:
        invalid_list = []
        for i in result:
            # policyRule must begin with VO:
            if not re.match('^VO:', i) and i != 'ALL' and i not in invalid_list:
                invalid_list.append(i)
        for i in invalid_list:
            result.remove(i)
        if invalid_list:
            result.append('INVALID')
    elif 'list' not in CONF:
        for i in result:
            # policyRule must begin with VO:
            if 'vo' in result[i]:
                if result[i]['vo'] != None:
                    for k in range(len(result[i]['vo'])):
                        if not re.match('^VO:', result[i]['vo'][k]) and result[i]['vo'][k] != 'ALL':
                            result[i]['vo'][k] = 'INVALID'

    if 'clean' in CONF or 'strict' in CONF:
        if 'list' in CONF:
            invalid_list = []
            for i in result:
                # all elements which contain spaces or are empty are invalid.
                if not i or i == 'None' or i.find(' ') >= 0:
                    invalid_list.append(i)
                # version should contain a number
                if CONF['list'] in ['intv', 'impv']:
                    if not re.match('^.*[0-9]+', i) and i not in invalid_list:
                        invalid_list.append(i)
                # quality level is restricted
                if CONF['list'] == 'ql':
                    if i not in values['ql'] and i not in invalid_list:
                        invalid_list.append(i)
                # url must be correctly formated
                if CONF['list'] == 'url':
                    if not re.match('(?:(?:([a-z0-9+.-]+)://)|(www\.))+(([a-zA-Z0-9\._-]+\.[a-zA-Z]{2,6})|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}))(/[a-zA-Z0-9\&amp;%_\./-~-]*)?', i) and i not in invalid_list:
                        invalid_list.append(i)
                if 'strict' in CONF:
                    # "j" is restricted
                    for j in ['cap', 'type', 'int']:
                        if CONF['list'] == j:
                            if i not in values[j] and i not in invalid_list:
                                invalid_list.append(i)
            for i in invalid_list:
                result.remove(i)
            if invalid_list:
                result.append('INVALID')            
        else:
            for i in result:
                # all elements which contain spaces or are empty
                for j in result[i]:
                    if ATTRIBUTES[j][3]:
                        if result[i][j]:
                            for k in range(len(result[i][j])):
                                if not result[i][j][k] or result[i][j][k].find(' ') >= 0:
                                    result[i][j][k] = 'INVALID'
                            continue
                        else:
                            result[i][j] = ['INVALID']
                    elif not result[i][j] or result[i][j].find(' ') >= 0:
                        result[i][j] = 'INVALID'
                # version should contain a number
                for j in ['intv', 'impv']:
                    if j in result[i]:
                        if not re.match('^.*[0-9]+', result[i][j]):
                            result[i][j] = 'INVALID'
                # quality level is restricted
                if 'ql' in result[i]:
                    if result[i]['ql'] not in values['ql']:
                        result[i]['ql'] = 'INVALID'
                # url must be correctly formated
                if 'url' in result[i]:
                    if not re.match('(?:(?:([a-z0-9+.-]+)://)|(www\.))+(([a-zA-Z0-9\._-]+\.[a-zA-Z]{2,6})|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}))(/[a-zA-Z0-9\&amp;%_\./-~-]*)?', result[i]['url']):
                        result[i]['url'] = 'INVALID'
                if 'strict' in CONF:
                    if 'cap' in result[i]:
                        for k in range(len(result[i]['cap'])):
                            if result[i]['cap'][k] not in values['cap'] and result[i]['cap'][k] != 'INVALID':
                                result[i]['cap'][k] = 'INVALID'
                    for j in ['type', 'int']:
                        # "j" is restricted
                        if j in result[i]:
                            if result[i][j] not in values[j] and result[i][j] != 'INVALID':
                                result[i][j] = 'INVALID'

    #Renaming of attributes
    if 'list' not in CONF:
        result2 = {}
        for i in result:
            result2[i] = {}
            for j in result[i]:
                result2[i][ATTRIBUTES[j][1]+ATTRIBUTES[j][2]] = result[i][j]
        result = result2
    return result

def serialize_output(result):
    """Return the output with the wished format""" 

    if 'list' in CONF:
        output = '\n'.join(result)
    elif 'csv' in CONF:
        csv_list = []
        titles = []
        for j in CONF['args']:
           titles.append(ATTRIBUTES[j][1]+ATTRIBUTES[j][2]) 
        csv_list.append(','.join(titles))
        for i in result:
            tmp_list = []
            for j in CONF['args']:
                att = ATTRIBUTES[j][1]+ATTRIBUTES[j][2]
                if result[i][att] == None:
                    tmp_list.append('None')
                elif ATTRIBUTES[j][3]:
                    tmp_list.append('"'+','.join(result[i][att])+'"')
                else:
                    tmp_list.append(result[i][att])
            csv_list.append(','.join(tmp_list))
        output = '\n'.join(csv_list)
    elif 'json' in CONF:
        output = json.dumps(result)
    elif 'emi' in CONF:
        emi_list = []
        for i in result:
            dic_tmp = {}
            for j in CONF['args']:
                att = ATTRIBUTES[j][1]+ATTRIBUTES[j][2]
		dic_tmp[ATTRIBUTES[j][4]] = result[i][att]
            emi_list.append(json.dumps(dic_tmp))
        output = '['+', '.join(emi_list)+']'
    else:
        entry_list = []
        for i in result:
            for j in CONF['args']:
                att = ATTRIBUTES[j][1]+ATTRIBUTES[j][2]
                if result[i][att] == None:
                    entry_list.append(att+': None')
                elif ATTRIBUTES[j][3]:
                    entry_list.append(att+': '+', '.join(result[i][att]))
                else:
                    entry_list.append(att+': '+result[i][att])
            entry_list.append('')
        output = '\n'.join(entry_list)
    return output

if __name__ == "__main__":
    main(sys.argv[1:])
