Commit ba55251d authored by Agustin Pane's avatar Agustin Pane
Browse files

Merge branch 'development' into 'master'

(Feature) - Release v2.0.0

See merge request !7
parents 7465c542 c822c46c
BACKEND_URL=https://svmon.eudat.eu:8443/api/serviceComponent/jsonreport
CERTIFICATE_PATH=
TOKEN_PATH=
DEBUG_MODE=false
## PySVMON
PySVMON is a develoment project for SVMON client. It migrates bash-based client code to python.
The development is based on python 2.7.13, and should also run in Python3.0.
The development is based on python 2.7.13 but runs also in Python3.0.
One has two options to install the client.
1. By root privilege.
2. In virtualenv. In such a case, the report can be only sent via --send options; we are searching for solutions
......@@ -23,7 +23,7 @@ Then one can use pip tools directly ( with root privileges ),
<b>Update pip client to the latest version for Python2.7 (pip 20.3.4 ) </b>
pip install --upgrade pip
<b>NOTE:<b/>
Please take in mind that svmon won't work with pip versions highers than 20.3.4 or with Python3. We are working in make it compatible with new versions of python but for the moment you will need Python2.7 and pip 20.3.4.
......@@ -31,6 +31,10 @@ For instructions about how to install pip 20.3.4, please check [this](https://gi
----
Install python-dotenv:
pip install python-dotenv
For test, just run the command svmon, the shell will return you help information. One can also try
svmon -T (--test)
......
PySVMON
PySVMON is a develoment project for SVMON client. It migrates bash-based client code to python. The development is based on python 2.7.13, and shall be tested on python 3.* version(TO DO).
PySVMON is a develoment project for SVMON client. It migrates bash-based client code to python. The development is based on python 2.7.13 but also runs on python 3.* version.
Before installation, check your installation of pakiti-client, see whether it support --svmonreport option.
......@@ -9,10 +9,14 @@ Then one can use pip tools directly (root privileges if no virtualenv),
pip install svmon-client
Update pip client to the latest version (pip 18.1 )
Update pip client to the latest version (pip 21.1.3)
pip install --upgrade pip
Install python-dotenv:
pip install python-dotenv
For test, just run the command svmon, the shell will return you help information. One can also try
svmon -T (--test)
......
This diff is collapsed.
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from setuptools import setup
from setuptools import setup, find_packages
setup(
name='svmon-client',
version='1.3.7-3',
setup(
name='svmon_python_client',
version='2.2.4',
packages=['svmon_client'],
scripts=['svmon','pakiti-client'],
scripts=['svmon'],
package_data={'' : ['*.json','*.pem']},
description='This is a python implementation of SVMON client',
long_description=open('README.rst').read(),
long_description=open('README.rst').read(),
author='Jie Yuan',
author_email='jie.yuan@kit.edu',
maintainer='Jie Yuan',
maintainer_email='jie.yuan@kit.edu',
maintainer='Agustin Pane',
maintainer_email='agustin.pane@kit.edu',
license='MIT License',
platforms=["all"],
url='https://gitlab.eudat.eu/jie.yuan/pysvmon',
classifiers=[
'Development Status :: 4 - Beta',
url='https://gitlab.eudat.eu/EUDAT-TOOLS/SVMON/pysvmon',
classifiers=[
'Development Status :: 4 - Beta',
'Operating System :: OS Independent',
'Intended Audience :: Developers',
'License :: OSI Approved :: MIT License',
......
......@@ -11,7 +11,7 @@ from svmon_client import services
from svmon_client import json_operations
def main(argv):
try:
opts, args = getopt.getopt(argv,'h:t:S:THVpd:',['help','version','site=','test',\
'host=','tag=','print','dump','type=','list-service-type','show-config','send',\
......@@ -104,7 +104,7 @@ def main(argv):
save = 0
exit(-1)
if save == 1:
print("Saving configurations ....")
if ( handle_server_path ==""):
......@@ -136,11 +136,11 @@ def main(argv):
svmonreport.send_report_to_svmon_server()
exit(0)
def help_info():
return '''
return '''
SVMON client -- Service Versions Monitoring
SVMON collects information of softwares in EUDAT services
and their components. SVMON client supports the following
......@@ -148,10 +148,10 @@ def help_info():
-H or --help Help information
-V or --version Show SVMON client version
-S or --site Declear the host site
-T or --test Test the installation of SVMON client
-S or --site Declear the host site
-h or --host Declear the host name
-t or --type Declear the service type.
-t or --type Declear the service type.
If type is "b2handle", please use --epic-version-file, --handle-server-path
to let client know how to parse the versions.
--component Specify the name of service component.
......@@ -161,15 +161,15 @@ def help_info():
information will be stored in component.json file.
-p or --print Print the collected info to the stardard output
-d or --dump Save configurations to a file in JSON format.
Defualt "svmon_client/config.json".
Default "svmon_client/config.json".
--list-service-type List all supported service types
--show-config Show the site,host, service type configurations.
--send Send the reports to SVMON server.
--handle-server-path Locate the directory where handle is installed when service type is "b2handle".
--token Add API token to post report to svmon.eudat.eu bypassing pakiti-client.
To get a token, one could go to the webpage; in the profile dialog, then create.
If you have problems with SVMON client usage, please contact us(jie.yuan@kit.edu).
--token Add API token to post report to svmon.eudat.eu
To get a token, one could go to the webpage (svmon.eudat.eu); in the profile dialog, then create.
If you have problems with SVMON client usage, please contact us at agustin.pane@kit.edu.
'''
......@@ -179,18 +179,18 @@ Currently, we are supporting following service types:
b2safe
gitlab
svmon
b2handle
b2access
b2share
svmon
b2handle
b2access
b2share
b2find --todo
b2drop
b2drop
dpmt --todo
eudat_website --todo
b2gether --todo
b2gether --todo
gocdb --todo
Please contact us(jie.yuan@kit.edu) if you plan to add other services.
Please contact us(agustin.pane@kit.edu) if you plan to add other services.
'''
def version_info():
......@@ -200,5 +200,5 @@ if __name__=="__main__":
import sys
# print sys.version
main(sys.argv[1:])
#
# spec file for package svmon
#
# Copyright (c) 2018 SUSE LINUX Products GmbH, Nuernberg, Germany.
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
# upon. The license for this file, and modifications and additions to the
# file, is the same license as for the pristine package itself (unless the
# license for the pristine package is not an Open Source License, in which
# case the license is the MIT License). An "Open Source License" is a
# license that conforms to the Open Source Definition (Version 1.9)
# published by the Open Source Initiative.
# Please submit bugfixes or comments via http://bugs.opensuse.org/
#
Name: svmon-client
Version: s1.0.1
Release: 1
License: GPL
Summary: svmon client
Url: https://gitlab.eudat.eu/jie.yuan/pysvmon
Group: EUDAT/SERVICE
Source: %{name}-%{version}.tar.gz
#Patch:
#BuildRequires: gcc make
#PreReq: python2.7
Provides: svmon service
BuildRoot: %{_tmppath}/%{name}-%{version}-build
PACKAGER: KIT
%description
The funtionality of the SVMON based on the scripts running at sites, which periodically collect the information on software versions of B2services and trasmit it furhter to the server in secure manner. Currently the procedure allows automatically extract the B2SAFE and iRODS versions, as well as versions of handle system and epic version, the programmatic extraction for other B2Services and collaborative tools is under study.
%prep
%setup -q
#%build
#./configure
#make %{?_smp_mflags}
%install
mkdir -p ${RPM_BUILD_ROOT}/opt/%{name}
cp -rf README.md ${RPM_BUILD_ROOT}/opt/%{name}
mkdir -p ${RPM_BUILD_ROOT}/usr/bin
cp -f svmon ${RPM_BUILD_ROOT}/usr/bin/svmon
cp -f pakiti-client ${RPM_BUILD_ROOT}/usr/bin/pakiti-client
mkdir -p ${RPM_BUILD_ROOT}/usr/lib/python2.7/site-packages/
cp -rf svmon_client ${RPM_BUILD_ROOT}/usr/lib/python2.7/site-packages/
#make install DESTDIR=%{buildroot} %{?_smp_mflags}
%post
%postun
%files
/opt/%{name}
/usr/bin/svmon
/usr/bin/pakiti-client
/usr/lib/python2.7/site-packages/svmon_client
%defattr(-,root,root)
%doc README.md
from dotenv import load_dotenv
import os
import requests as request
import json
class RemoteConfig:
config = {}
config['BACKEND_URL'] = 'https://svmon.eudat.eu/api/serviceComponent/jsonreport'
config['REMOTE_CONFIG_URL'] = 'https://svmon.eudat.eu/api/remoteConfig/'
config['DEBUG_MODE'] = True
def __init__(self):
load_dotenv()
self.loadEnvVars()
def loadRemoteConfig(self):
try:
headers = {}
headers['Content-Type'] = 'application/json'
url = self.config['REMOTE_CONFIG_URL']
apiToken = self.getUserApiToken()
if(apiToken):
headers['Authorization'] = 'Bearer ' + apiToken
url = url + 'user'
response = request.get(url, headers=headers)
responseBody = response.json()
if(response.status_code != 200):
if(self.config['DEBUG_MODE']):
print('Recover of remote config FAILED ', response.status_code)
print('Failed to load remote config, using default env vars...')
return False
else:
if(self.config['DEBUG_MODE']):
print('Recover of remote config finished with code ', response.status_code)
if('environments' in responseBody):
environments = responseBody["environments"]
if('backendUrl' in environments):
self.config["BACKEND_URL"] = environments["backendUrl"]
if('debugMode' in environments):
self.config["DEBUG_MODE"] = environments["debugMode"]
return True
except request.exceptions.RequestException as e:
if(self.config['DEBUG_MODE']):
print('Failed to load remote config, using default env vars...')
print(e)
return False
def getUserApiToken(self):
from . import services
cwd = os.path.dirname(services.__file__)
apiToken = None
filename = os.environ.get('TOKEN_PATH', cwd + "/token.json")
if os.path.exists(filename) == False:
return None
if os.access(filename, os.R_OK) == False:
return None
with open(filename, 'r') as f:
load_dict = json.load(f)
if 'token' in load_dict and load_dict.get('token') != '':
apiToken = load_dict.get('token')
else:
return None
self.config['TOKEN'] = apiToken
return apiToken
def loadEnvVars(self):
remoteEnvVars = self.loadRemoteConfig()
if(remoteEnvVars and self.config['DEBUG_MODE']):
print("Remote environment variables loaded: ")
print(self.config)
else:
backendUrl = os.environ.get('BACKEND_URL', self.config['BACKEND_URL'])
self.config['BACKEND_URL'] = backendUrl
debugMode = os.environ.get('DEBUG_MODE', self.config['DEBUG_MODE'])
self.config['DEBUG_MODE'] = debugMode
return self.config
def getConfig(self):
return self.config
......@@ -6,16 +6,20 @@ import json
import subprocess
from . import services
from . import json_operations
from dotenv import load_dotenv
from svmon_client import remoteConfig
class SVMONReport:
def __init__(self):
load_dotenv()
self.site=""
self.host=""
self.service_names=[]
self.service_type=""
self.operating_system=""
self.operating_system=""
self.tags=[]
self.config = remoteConfig.RemoteConfig().getConfig()
cwd = os.path.dirname(services.__file__)
filename = cwd + "/config.json"
if os.path.exists(filename) == False:
......@@ -31,13 +35,12 @@ class SVMONReport:
self.site=str(load_dict['site'])
self.host=str(load_dict['host'])
self.service_type=str(load_dict['service_type'])
def refresh_service_name_and_tag(self):
if services.in_service_list(self.service_type) == False:
print("service type not configured")
if self.service_type == "b2access" or self.service_type == "b2drop" or self.service_type == "b2share":
cwd = os.path.dirname(services.__file__)
filename = cwd + "/components.json"
......@@ -46,19 +49,23 @@ class SVMONReport:
print("You have no access to read config file: components.json")
exit(-1)
with open(filename, 'r') as f:
load_dict = json.load(f)
if load_dict.has_key('components') and load_dict.has_key('versions') :
self.set_service_names(load_dict['components'])
self.set_tags(load_dict['versions'])
try:
load_dict = json.load(f)
if 'components' in load_dict and 'versions' in load_dict:
self.set_service_names(load_dict['components'])
self.set_tags(load_dict['versions'])
if len(self.service_names) != len(self.tags):
print("The length of the components is not equal to the versions")
exit(-1)
if len(self.service_names) != len(self.tags):
print("The length of the components is not equal to the versions")
exit(-1)
else:
except ValueError:
print("Your components.json file is not valid, please setup your configuration again or contact us for support via svmon --help")
exit(-1)
else:
self.service_names = services.get_service_name(self.service_type)
b2handle_config= {}
if self.service_type == 'b2handle' or self.service_type == "B2HANDLE":
cwd = os.path.dirname(services.__file__)
......@@ -69,7 +76,7 @@ class SVMONReport:
exit(-1)
with open(filename, 'r') as f:
load_dict= json.load(f)
if load_dict.has_key('handle_server_path'):
if 'handle_server_path' in load_dict:
if load_dict.get('handle_server_path') != None and load_dict.get('handle_server_path') != '':
b2handle_config['handle_server_path'] = load_dict.get('handle_server_path')
self.tags = services.get_service_tag(self.service_type,b2handle_config)
......@@ -79,7 +86,7 @@ class SVMONReport:
else:
self.tags = services.get_service_tag(self.service_type)
def get_site(self):
return self.site
......@@ -94,7 +101,7 @@ class SVMONReport:
def get_tags(self):
return self.tags
def set_site(self,site):
if isinstance(site,str):
self.site=site
......@@ -114,7 +121,7 @@ class SVMONReport:
def set_tags(self,tags):
if isinstance(tags,list):
self.tags=tags
def set_operating_system(self,operating_system):
if isinstance(operating_system,str):
self.operating_system=operating_system
......@@ -122,7 +129,7 @@ class SVMONReport:
def set_service_names_and_tags(self,service_name,tag):
this.service_names.append(service_name)
this.tags.append(tag)
def print_report(self):
if len(self.service_names) > 0:
......@@ -159,7 +166,7 @@ class SVMONReport:
else:
print("No services to be jsonified")
return {}
def save_b2handle_config_to_json(self,handle_server_path=None):
if handle_server_path == None :
print("No b2handle configuration input")
......@@ -205,7 +212,7 @@ class SVMONReport:
filename = cwd + "/config.json"
with open(filename, 'r') as f:
load_dict= json.load(f)
if load_dict.has_key('handle_server_path') and load_dict.get('handle_server_path') != None and load_dict.get('handle_server_path') != "":
if 'handle_server_path' in load_dict and load_dict.get('handle_server_path') != None and load_dict.get('handle_server_path') != "":
res['handle_server_path'] = load_dict.get('handle_server_path')
cwd = os.path.dirname(services.__file__)
......@@ -257,9 +264,9 @@ class SVMONReport:
else:
print("Successfully saved component data")
exit(1)
def set_pair(self,key,value):
if isinstance(key,str) == False:
......@@ -305,9 +312,10 @@ class SVMONReport:
def send_report_to_svmon_server(self):
import requests as re
print('Sending report to svmon server...')
cwd = os.path.dirname(services.__file__)
mytoken = ""
filename = cwd + "/token.json"
filename = os.environ.get('TOKEN_PATH', cwd + "/token.json")
if os.path.exists(filename) == False:
print("You have no token configured, please go to svmon.eudat.eu webpage to create a token and configure it")
exit(-1)
......@@ -317,21 +325,25 @@ class SVMONReport:
exit(-1)
with open(filename, 'r') as f:
load_dict = json.load(f)
if load_dict.has_key('token') and load_dict.get('token') != '':
if 'token' in load_dict and load_dict.get('token') != '':
mytoken = load_dict.get('token')
else:
print("Please configure api token")
exit(1)
url = 'https://svmon.eudat.eu:8443/api/serviceComponent/jsonreport'
url = self.config['BACKEND_URL']
print("BACKEND_URL", url)
headers = {}
headers['Content-Type'] = 'application/json'
headers['Authorization'] = 'Bearer ' + mytoken
certs = cwd + "/chain_TERENA_SSL_CA_3.pem"
cert = os.environ.get('CERTIFICATE_PATH', cwd + "/chain_TERENA_SSL_CA_3.pem")
if(not len(self.service_names)):
print('No services available, sending report failed')
exit(1)
enableDebug = self.config['DEBUG_MODE']
postErrors = 0
for i in range(len(self.service_names)):
res = {}
res['siteName']=self.site
......@@ -340,15 +352,38 @@ class SVMONReport:
res['serviceType']=self.service_type
res['serviceComponentName']=self.service_names[i]
res['tagAtSite']=self.tags[i]
r = re.post(url, headers=headers, data=json.dumps(res), verify=False)
if (r.status_code != 201):
if(enableDebug):
print('Going to send report: ', res, ' to endpoint: ',url)
print('Certificate path: ', cert)
try:
r = re.post(url, data=json.dumps(res), headers=headers, verify=cert)
#r = re.post(url, headers=headers, data=json.dumps(res), verify=False)
print('Request to svmon server finished with status code:', r.status_code)
if(r.status_code == 401):
print('Sending report failed, user not authorized, did you provided a valid token?, please check your token.json')
exit(1)
if(r.status_code == 404):
print('Service ', self.service_type,' does not exists, please check the svmon config')
postErrors +=1
continue
if (r.status_code != 201 and r.status_code != 200):
print('Sending report failed, please check your configuration')
exit(1)
except re.exceptions.RequestException as e:
print('Sending report failed, host offline?')
exit(1)
if(postErrors > 0):
if(postErrors == len(self.service_names)):
print('Sending report failed, please check your configuration')
exit(1)
print('Sending report finished with some failed reports: ${postErrors}/${len(self.service_names)}, please check your configuration')
exit(1)
print('Report has been sent successfully.')
exit(0)
def save_token(self,token):
print('Saving token...')
if token == "":
print("No token to be saved")
exit(0)
......
......@@ -3,6 +3,8 @@
import subprocess
import os
from platform import python_version
from svmon_client import remoteConfig
_services={}
......@@ -49,36 +51,21 @@ def get_service_name(service_type):
def get_service_tag(service_type,configs=None):
print("Get service tag called...")
tags=[]
if service_type == None or service_type == "" or isinstance(service_type,str) == False:
print("The service type argument should be a non-empty string")
exit(1)
if in_service_list(service_type) == False:
print("The service type is currently unsupported in svmon client")
exit(1)
tags=[]
if service_type == "svmon_client":
tmp = subprocess.Popen('pip show svmon-client', shell=True, stdout = subprocess.PIPE)
tmp = subprocess.Popen('grep Version', shell = True, stdin=tmp.stdout, stdout = subprocess.PIPE)
tmp = tmp.communicate()
tmp=tmp[0]
if tmp == '' or tmp == None:
print("No svmon client has been installed, please refer to https://gitlab.eudat.eu/jie.yuan/pysvmon")
exit(1)
ltmp=tmp.split('\n')
ltmp=ltmp[0].split(':')
if ltmp == None or len(ltmp) < 2:
print("SVMON client can not be resolved, please check for installation")
exit(1)
ltmp=ltmp[1]
ltmp.replace("\n","")
ltmp.replace(" ", "")
if ltmp == "":
print("the version of svmon client can not resolved. Please check installation of svmon client")
exit(1)
tags.append(ltmp)
print("Getting tag for svmon_client")
tags.append(get_version())
return tags
elif service_type == "gitlab":