1271 lines
49 KiB
Python
1271 lines
49 KiB
Python
###
|
|
# Copyright 2020 Hewlett Packard Enterprise, Inc. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
###
|
|
# -*- coding: utf-8 -*-
|
|
"""Monolith database implementation. Crawls Redfish and Legacy REST implementations
|
|
and holds all data retrieved. The created database is called the **monolith** and referenced as
|
|
such in the code documentation."""
|
|
|
|
# ---------Imports---------
|
|
|
|
import logging
|
|
import re
|
|
import sys
|
|
import threading
|
|
from re import error as regexerr
|
|
|
|
# Added for py3 compatibility
|
|
import six
|
|
|
|
try:
|
|
from collections import OrderedDict, defaultdict
|
|
except ImportError:
|
|
from collections.abc import OrderedDict, defaultdict
|
|
|
|
from queue import Queue
|
|
|
|
import jsonpath_rw
|
|
import jsonpointer
|
|
from jsonpointer import set_pointer
|
|
from six.moves.urllib.parse import urlparse, urlunparse
|
|
from six.moves.urllib.parse import ParseResult
|
|
from six.moves.urllib.parse import quote
|
|
|
|
from redfish.rest.containers import RestRequest, StaticRestResponse
|
|
from redfish.ris.ris_threaded import LoadWorker
|
|
from redfish.ris.sharedtypes import Dictable
|
|
|
|
# ---------End of imports---------
|
|
|
|
# ---------Debug logger---------
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
|
|
# ---------End of debug logger---------
|
|
|
|
|
|
class BiosUnregisteredError(Exception):
|
|
"""Raised when BIOS has not been registered correctly in iLO"""
|
|
|
|
pass
|
|
|
|
|
|
class SchemaValidationError(Exception):
|
|
"""Schema Validation Class Error"""
|
|
|
|
pass
|
|
|
|
|
|
class SessionExpired(Exception):
|
|
"""Raised when session has expired"""
|
|
|
|
pass
|
|
|
|
|
|
class RisMonolithMemberBase(Dictable):
|
|
"""RIS monolith member base class"""
|
|
|
|
pass
|
|
|
|
|
|
class RisInstanceNotFoundError(Exception):
|
|
"""Raised when attempting to select an instance that does not exist"""
|
|
|
|
pass
|
|
|
|
|
|
class RisMonolithMemberv100(RisMonolithMemberBase):
|
|
"""Class used by :class:`RisMonolith` for holding information on a response and adds extra data
|
|
for monolith usage. A member can be marked as *modified* which means another operation may have
|
|
rendered this member out of date. It should be reloaded before continuing to ensure data is
|
|
up to date.
|
|
|
|
:param restresp: `RestResponse` to create a member from.
|
|
:type restresp: :class:`redfish.rest.containers.RestResponse`
|
|
:param isredfish: Flag if the response is redfish or not
|
|
:type isredfish: bool
|
|
"""
|
|
|
|
def __init__(self, restresp=None, isredfish=True):
|
|
self._resp = restresp
|
|
self._patches = list()
|
|
# Check if typedef can be used here
|
|
self._typestring = "@odata.type" if isredfish else "Type"
|
|
self.modified = False
|
|
self.defpath = self.deftype = self.defetag = self._type = None
|
|
self.__bool__ = self.__nonzero__
|
|
|
|
@property
|
|
def type(self):
|
|
"""Get type of the monolith member's response"""
|
|
try:
|
|
if self and self._typestring in self.resp.dict:
|
|
return self.resp.dict[self._typestring]
|
|
# Added for object type
|
|
elif self and "type" in self.resp.dict:
|
|
return self.resp.dict["type"]
|
|
except (AttributeError, ValueError, TypeError):
|
|
return self.deftype # data not yet fetched, probably empty dict, so assume deftype
|
|
return None
|
|
|
|
@property
|
|
def maj_type(self):
|
|
"""Get major type of the monolith member's response"""
|
|
if self.type:
|
|
if "." in self.type:
|
|
types = ".".join(self.type.split(".", 2)[:2])
|
|
retval = types[1:] if types.startswith("#") else types
|
|
else:
|
|
retval = self.type
|
|
return retval
|
|
return self.deftype
|
|
|
|
def __nonzero__(self):
|
|
"""Defining the bool value for the class"""
|
|
return True if self.resp else False
|
|
|
|
@property
|
|
def resp(self):
|
|
"""Get the entire response of the monolith member"""
|
|
return self._resp
|
|
|
|
@property
|
|
def path(self):
|
|
"""Get path of the monolith member's response"""
|
|
|
|
try:
|
|
if self:
|
|
return self.resp.request.path
|
|
except (AttributeError, ValueError):
|
|
pass
|
|
return self.defpath
|
|
|
|
@property
|
|
def patches(self):
|
|
"""Get patches for the monolith member"""
|
|
return self._patches
|
|
|
|
@patches.setter
|
|
def patches(self, val):
|
|
"""Set patches for the monolith member"""
|
|
self._patches = val
|
|
|
|
@property
|
|
def dict(self):
|
|
"""Get the dictionary of the monolith member's response"""
|
|
return self._resp.dict
|
|
|
|
@property
|
|
def etag(self):
|
|
"""Get the etag of the response"""
|
|
return self.defetag if not self.resp else self.resp.getheader("etag")
|
|
|
|
def popdefs(self, typename, pathval, etagval):
|
|
"""Populate the default values in the class
|
|
|
|
:param typename: The default **Type** string. Example: @odata.type
|
|
:type typename: str
|
|
:param pathval: The default **Path** string. Example: @odata.id
|
|
:type pathval: str
|
|
:param etagval: The default **ETag** value.
|
|
:type etagval: str
|
|
"""
|
|
self.defetag = etagval
|
|
self.deftype = typename
|
|
self.defpath = pathval
|
|
|
|
def to_dict(self):
|
|
"""Converts Monolith Member to a dictionary. This is the reverse of :func:`load_from_dict`.
|
|
|
|
:returns: returns the Monolith Member in dictionary form
|
|
"""
|
|
result = OrderedDict()
|
|
if self.maj_type:
|
|
result["Type"] = self.type
|
|
|
|
if self.resp:
|
|
if self.maj_type == "Collection.1" and "MemberType" in self.resp.dict:
|
|
result["MemberType"] = self.resp.dict["MemberType"]
|
|
|
|
result["links"] = OrderedDict()
|
|
result["links"]["href"] = ""
|
|
result["ETag"] = self.etag
|
|
|
|
if self.resp:
|
|
result["Content"] = self.resp.dict
|
|
result["Status"] = self.resp.status
|
|
result["Headers"] = self.resp.getheaders()
|
|
result["OriginalUri"] = self.path
|
|
result["Patches"] = self._patches
|
|
result["modified"] = self.modified
|
|
result["MajType"] = self.maj_type
|
|
|
|
return result
|
|
|
|
def load_from_dict(self, src):
|
|
"""Load variables to a monolith member from a dictionary.
|
|
This is the reverse of :func:`to_dict`.
|
|
|
|
:param src: Source to load member data from.
|
|
:type src: dict
|
|
"""
|
|
if "Type" in src:
|
|
self._type = src["Type"]
|
|
if "Content" in src:
|
|
restreq = RestRequest(method="GET", path=src["OriginalUri"])
|
|
src["restreq"] = restreq
|
|
self._resp = StaticRestResponse(**src)
|
|
self.deftype = src["MajType"]
|
|
self.defpath = src["OriginalUri"]
|
|
self.defetag = src["ETag"]
|
|
self._patches = src["Patches"]
|
|
self.modified = src["modified"]
|
|
|
|
|
|
class RisMonolith(Dictable):
|
|
"""Monolithic cache of RIS data. This takes a :class:`redfish.rest.v1.RestClient` and uses it to
|
|
gather data from a server and saves it in a modifiable database called monolith.
|
|
|
|
:param client: client to use for data retrieval.
|
|
:type client: :class:`redfish.rest.v1.RestClient`
|
|
:param typepath: The compatibility class to use for differentiating between Redfish/LegacyRest.
|
|
:type typepath: :class:`redfish.rest.ris.Typesandpathdefines`
|
|
:param directory_load: The flag to quick load using resource directory if possible.
|
|
When set to True this will load paths, etags, and types, but not create responses for
|
|
every monolith member. When responses are needed, they will need to be loaded separately.
|
|
:type directory_load: bool
|
|
"""
|
|
|
|
def __init__(self, client, typepath, directory_load=True):
|
|
self._client = client
|
|
self.name = "Monolithic output of RIS Service"
|
|
self._visited_urls = list()
|
|
self._type = None
|
|
self._name = None
|
|
self.progress = 0
|
|
self._directory_load = directory_load
|
|
self.is_redfish = self.client.is_redfish
|
|
self.typesadded = defaultdict(set)
|
|
self.paths = dict()
|
|
self.ctree = defaultdict(set)
|
|
self.colltypes = defaultdict(set)
|
|
|
|
self.typepath = typepath
|
|
self.collstr = self.typepath.defs.collectionstring
|
|
self.etagstr = "ETag"
|
|
if self.is_redfish:
|
|
self._resourcedir = "/redfish/v1/ResourceDirectory/"
|
|
else:
|
|
self._resourcedir = "/rest/v1/ResourceDirectory"
|
|
|
|
# MultiThreading
|
|
self.get_queue = Queue()
|
|
self.threads = []
|
|
|
|
@property
|
|
def directory_load(self):
|
|
"""The flag to gather information about a tree without downloading every path. Only usable
|
|
on HPE systems with a ResourceDirectory. type"""
|
|
return self._directory_load
|
|
|
|
@directory_load.setter
|
|
def directory_load(self, dir_load):
|
|
"""Set the directory_load flag"""
|
|
self._directory_load = dir_load
|
|
|
|
@property
|
|
def type(self):
|
|
"""Return monolith version type"""
|
|
return "Monolith.1.0.0"
|
|
|
|
@property
|
|
def visited_urls(self):
|
|
"""The urls visited by the monolith"""
|
|
return list(set(self._visited_urls) | set(self.paths.keys()))
|
|
|
|
@visited_urls.setter
|
|
def visited_urls(self, visited_urls):
|
|
"""Set visited URLS."""
|
|
self._visited_urls = visited_urls
|
|
|
|
@property
|
|
def types(self):
|
|
"""Returns list of types for members in the monolith
|
|
|
|
:rtype: list
|
|
"""
|
|
return list(self.typesadded.keys())
|
|
|
|
@types.setter
|
|
def types(self, member):
|
|
"""Adds a member to monolith
|
|
|
|
:param member: Member created based on response.
|
|
:type member: RisMonolithMemberv100
|
|
"""
|
|
self.typesadded[member.maj_type].add(member.path)
|
|
patches = []
|
|
if member.path in list(self.paths.keys()):
|
|
patches = self.paths[member.path].patches
|
|
self.paths[member.path] = member
|
|
self.paths[member.path].patches.extend([patch for patch in patches])
|
|
|
|
@property
|
|
def client(self):
|
|
"""Returns the current client object reference
|
|
|
|
:rtype: class object
|
|
"""
|
|
return self._client
|
|
|
|
@client.setter
|
|
def client(self, curr_client):
|
|
"""Set the current client
|
|
|
|
:param curr_client: current client object
|
|
:type curr_client: class object
|
|
"""
|
|
self._client = curr_client
|
|
|
|
def path(self, path):
|
|
"""Provides the member corresponding to the path specified. Case sensitive.
|
|
|
|
:param path: path of the monolith member to return
|
|
:type path: str
|
|
:rtype: RisMonolithMemberv100
|
|
"""
|
|
try:
|
|
return self.paths[path]
|
|
except:
|
|
return None
|
|
|
|
def iter(self, typeval=None):
|
|
"""An iterator that yields each member of monolith associated with a specific type. In the
|
|
case that no type is included this will yield all members in the monolith.
|
|
|
|
:rtype: RisMonolithMemberv100
|
|
"""
|
|
if not typeval:
|
|
for _, val in list(self.paths.items()):
|
|
yield val
|
|
else:
|
|
for typename in self.gettypename(typeval):
|
|
for item in self.typesadded[typename]:
|
|
yield self.paths[item]
|
|
|
|
# types = next(self.gettypename(typeval), None)
|
|
# if types in self.typesadded:
|
|
# for item in self.typesadded[types]:
|
|
# yield self.paths[item]
|
|
# else:
|
|
# raise RisInstanceNotFoundError("Unable to locate instance for" \
|
|
# " '%s'\n" % typeval)
|
|
|
|
def itertype(self, typeval):
|
|
"""Iterator that yields member(s) of given type in the monolith and raises an error if no
|
|
member of that type is found.
|
|
|
|
:param typeval: type name of the requested member.
|
|
:type typeval: str
|
|
:rtype: RisMonolithMemberv100
|
|
"""
|
|
typeiter = self.gettypename(typeval)
|
|
types = next(typeiter, None)
|
|
if types:
|
|
while types:
|
|
for item in self.typesadded[types]:
|
|
yield self.paths[item]
|
|
types = next(typeiter, None)
|
|
else:
|
|
raise RisInstanceNotFoundError("Unable to locate instance for '%s'\n" % typeval)
|
|
|
|
def typecheck(self, types):
|
|
"""Check if a member of given type exists in the monolith
|
|
|
|
:param types: type to check.
|
|
:type types: str
|
|
:rtype: bool
|
|
"""
|
|
if any(types in val for val in self.types):
|
|
return True
|
|
return False
|
|
|
|
def gettypename(self, types):
|
|
"""Takes a full type response and returns all major types associated.
|
|
Example: #Type.v1_0_0.Type will return iter(Type.1)
|
|
|
|
:param types: The type of the requested response.
|
|
:type types: str
|
|
:rtype: iter of major types
|
|
"""
|
|
types = types[1:] if types[0] in ("#", "#") else types
|
|
return iter((xt for xt in self.types if xt and types.lower() in xt.lower()))
|
|
|
|
def update_member(self, member=None, resp=None, path=None, init=True):
|
|
"""Adds member to the monolith. If the member already exists the
|
|
data is updated in place. Takes either a RisMonolithMemberv100 instance or a
|
|
:class:`redfish.rest.containers.RestResponse` along with that responses path.
|
|
|
|
:param member: The monolith member to add to the monolith.
|
|
:type member: RisMonolithMemberv100
|
|
:param resp: The rest response to add to the monolith.
|
|
:type resp: :class:`redfish.rest.containers.RestResponse`
|
|
:param path: The path correlating to the response.
|
|
:type path: str
|
|
:param init: Flag if addition is part of the initial load. Set this to false if you are
|
|
calling this by itself.
|
|
:type init: bool
|
|
"""
|
|
if not member and resp and path:
|
|
self._visited_urls.append(path.lower())
|
|
|
|
member = RisMonolithMemberv100(resp, self.is_redfish)
|
|
if not member: # Assuming for lack of member and not member.type
|
|
return
|
|
if not member.type:
|
|
member.deftype = "object" # Hack for general schema with no type
|
|
|
|
self.types = member
|
|
|
|
if init:
|
|
self.progress += 1
|
|
if LOGGER.getEffectiveLevel() == 40:
|
|
self._update_progress()
|
|
|
|
def load(
|
|
self,
|
|
path=None,
|
|
includelogs=False,
|
|
init=False,
|
|
crawl=True,
|
|
loadtype="href",
|
|
loadcomplete=False,
|
|
path_refresh=False,
|
|
json_out=False,
|
|
single=True,
|
|
):
|
|
"""Walks the entire data model and caches all responses or loads an individual path into
|
|
the monolith. Supports both threaded and sequential crawling.
|
|
|
|
:param path: The path to start the crawl from the provided path if crawling or
|
|
loads the path into monolith. If path is not included, crawl will start with
|
|
the default. The default is */redfish/v1/* or */rest/v1* depending on if the
|
|
system is Redfish or LegacyRest.
|
|
:type path: str.
|
|
:param includelogs: Flag to determine if logs should be downloaded as well in the crawl.
|
|
:type includelogs: bool
|
|
:param init: Flag to determine if this is the initial load.
|
|
:type init: bool
|
|
:param crawl: Flag to determine if load should crawl through found links.
|
|
:type crawl: bool
|
|
:param loadtype: Flag to determine if loading standard links: *href* or schema links: *ref*.
|
|
:type loadtype: str.
|
|
:param loadcomplete: Flag to download the entire data model including registries and
|
|
schemas.
|
|
:type loadcomplete: bool
|
|
:param path_refresh: Flag to reload the path specified, clearing any patches and overwriting the
|
|
current data in the monolith.
|
|
:type path_refresh: bool
|
|
"""
|
|
if init:
|
|
if LOGGER.getEffectiveLevel() >= 20 and not json_out:
|
|
sys.stdout.write("Discovering data...")
|
|
else:
|
|
LOGGER.info("Discovering data...")
|
|
self.name = self.name + " at %s" % self.client.base_url
|
|
|
|
selectivepath = path
|
|
if not selectivepath:
|
|
selectivepath = self.client.default_prefix
|
|
if loadtype == "href" and not self.client.base_url.startswith("blobstore://.") and not single:
|
|
if not self.threads:
|
|
for _ in range(6):
|
|
workhand = LoadWorker(self.get_queue)
|
|
workhand.setDaemon(True)
|
|
workhand.start()
|
|
self.threads.append(workhand)
|
|
|
|
self.get_queue.put(
|
|
(
|
|
selectivepath,
|
|
includelogs,
|
|
loadcomplete,
|
|
crawl,
|
|
path_refresh,
|
|
init,
|
|
None,
|
|
None,
|
|
self,
|
|
)
|
|
)
|
|
self.get_queue.join()
|
|
|
|
# Raise any errors from threads, and set them back to None after
|
|
excp = None
|
|
for thread in self.threads:
|
|
if excp is None:
|
|
excp = thread.get_exception()
|
|
thread.exception = None
|
|
|
|
if excp:
|
|
raise excp
|
|
|
|
# self.member_queue.join()
|
|
else:
|
|
# We can't load ref or local client in a threaded manner
|
|
self._load(
|
|
selectivepath,
|
|
originaluri=None,
|
|
crawl=crawl,
|
|
includelogs=includelogs,
|
|
init=init,
|
|
loadtype=loadtype,
|
|
loadcomplete=loadcomplete,
|
|
path_refresh=path_refresh,
|
|
prevpath=None,
|
|
)
|
|
|
|
if init:
|
|
if LOGGER.getEffectiveLevel() >= 20 and not json_out:
|
|
sys.stdout.write("Done\n")
|
|
else:
|
|
LOGGER.info("Done\n")
|
|
if self.directory_load and init:
|
|
self._populatecollections()
|
|
|
|
def loadallconfig(
|
|
self,
|
|
path=None,
|
|
includelogs=False,
|
|
init=False,
|
|
crawl=True,
|
|
loadtype="href",
|
|
loadcomplete=False,
|
|
path_refresh=False,
|
|
json_out=False,
|
|
single=True,
|
|
):
|
|
"""Walks the entire data model and caches all responses or loads an individual path into
|
|
the monolith. Supports both threaded and sequential crawling.
|
|
|
|
:param path: The path to start the crawl from the provided path if crawling or
|
|
loads the path into monolith. If path is not included, crawl will start with
|
|
the default. The default is */redfish/v1/* or */rest/v1* depending on if the
|
|
system is Redfish or LegacyRest.
|
|
:type path: str.
|
|
:param includelogs: Flag to determine if logs should be downloaded as well in the crawl.
|
|
:type includelogs: bool
|
|
:param init: Flag to determine if this is the initial load.
|
|
:type init: bool
|
|
:param crawl: Flag to determine if load should crawl through found links.
|
|
:type crawl: bool
|
|
:param loadtype: Flag to determine if loading standard links: *href* or schema links: *ref*.
|
|
:type loadtype: str.
|
|
:param loadcomplete: Flag to download the entire data model including registries and
|
|
schemas.
|
|
:type loadcomplete: bool
|
|
:param path_refresh: Flag to reload the path specified, clearing any patches and overwriting the
|
|
current data in the monolith.
|
|
:type path_refresh: bool
|
|
"""
|
|
|
|
selectivepath = path
|
|
if not selectivepath:
|
|
selectivepath = self.client.default_prefix
|
|
self._load(
|
|
selectivepath,
|
|
originaluri=None,
|
|
crawl=crawl,
|
|
includelogs=includelogs,
|
|
init=init,
|
|
loadtype=loadtype,
|
|
loadcomplete=loadcomplete,
|
|
path_refresh=path_refresh,
|
|
prevpath=None,
|
|
)
|
|
|
|
if init:
|
|
if LOGGER.getEffectiveLevel() >= 20 and not json_out:
|
|
sys.stdout.write("Done\n")
|
|
else:
|
|
LOGGER.info("Done\n")
|
|
if self.directory_load and init:
|
|
self._populatecollections()
|
|
|
|
def _load(
|
|
self,
|
|
path,
|
|
crawl=True,
|
|
originaluri=None,
|
|
includelogs=False,
|
|
init=True,
|
|
loadtype="href",
|
|
loadcomplete=False,
|
|
path_refresh=False,
|
|
prevpath=None,
|
|
):
|
|
"""Sequential version of loading monolith and parsing schemas.
|
|
|
|
:param path: path to start load from.
|
|
:type path: str
|
|
:param crawl: flag to determine if load should traverse found links.
|
|
:type crawl: bool
|
|
:param originaluri: variable to assist in determining originating path.
|
|
:type originaluri: str
|
|
:param includelogs: flag to determine if logs should be downloaded also.
|
|
:type includelogs: bool
|
|
:param init: flag to determine if first run of load.
|
|
:type init: bool
|
|
:param loadtype: flag to determine if load is meant for only href items.
|
|
:type loadtype: str.
|
|
:param loadcomplete: flag to download the entire monolith
|
|
:type loadcomplete: bool
|
|
:param path_refresh: flag to reload the members in the monolith instead of skip if they exist.
|
|
:type path_refresh: bool
|
|
"""
|
|
|
|
if (path.endswith("?page=1") or path.endswith(".json")) and not loadcomplete:
|
|
# Don't download schemas in crawl unless we are loading absolutely everything
|
|
return
|
|
elif path == "/" or "telemetry" in path.lower() or not path.startswith("/redfish/v1"):
|
|
return
|
|
elif not includelogs:
|
|
# Only include logs when asked as there can be an extreme amount of entries
|
|
if "/log" in path.lower():
|
|
return
|
|
|
|
# remove fragments and quote characters in path
|
|
p = urlparse(path)
|
|
p = ParseResult(
|
|
scheme=p.scheme, netloc=p.netloc, path=quote(p.path), params=p.params, query=p.query, fragment=""
|
|
)
|
|
path = urlunparse(p)
|
|
|
|
if prevpath and prevpath != path:
|
|
self.ctree[prevpath].update([path])
|
|
if not path_refresh:
|
|
if path.lower() in self.visited_urls:
|
|
return
|
|
LOGGER.debug("_loading %s", path)
|
|
|
|
resp = self.client.get(path)
|
|
if resp.status != 200 and path.lower() == self.typepath.defs.biospath:
|
|
raise BiosUnregisteredError()
|
|
elif resp.status == 401:
|
|
raise SessionExpired("Invalid session. Please logout and " "log back in or include credentials.")
|
|
elif resp.status not in (201, 200):
|
|
self.removepath(path)
|
|
return
|
|
|
|
if loadtype == "ref":
|
|
try:
|
|
if resp.status in (201, 200):
|
|
self.update_member(resp=resp, path=path, init=init)
|
|
self._parse_schema(resp)
|
|
except jsonpointer.JsonPointerException:
|
|
raise SchemaValidationError()
|
|
|
|
self.update_member(resp=resp, path=path, init=init)
|
|
|
|
fpath = lambda pa, path: (
|
|
path if pa.endswith(self.typepath.defs.hrefstring) and pa.startswith((self.collstr, "Entries")) else None
|
|
)
|
|
|
|
if loadtype == "href":
|
|
# follow all the href attributes
|
|
if self.is_redfish:
|
|
jsonpath_expr = jsonpath_rw.parse("$..'@odata.id'")
|
|
else:
|
|
jsonpath_expr = jsonpath_rw.parse("$..href")
|
|
matches = jsonpath_expr.find(resp.dict)
|
|
|
|
if "links" in resp.dict and "NextPage" in resp.dict["links"]:
|
|
if originaluri:
|
|
next_link_uri = originaluri + "?page=" + str(resp.dict["links"]["NextPage"]["page"])
|
|
href = "%s" % next_link_uri
|
|
|
|
self._load(
|
|
href,
|
|
originaluri=originaluri,
|
|
includelogs=includelogs,
|
|
crawl=crawl,
|
|
init=init,
|
|
prevpath=None,
|
|
loadcomplete=loadcomplete,
|
|
)
|
|
else:
|
|
next_link_uri = path + "?page=" + str(resp.dict["links"]["NextPage"]["page"])
|
|
|
|
href = "%s" % next_link_uri
|
|
self._load(
|
|
href,
|
|
originaluri=path,
|
|
includelogs=includelogs,
|
|
crawl=crawl,
|
|
init=init,
|
|
prevpath=None,
|
|
loadcomplete=loadcomplete,
|
|
)
|
|
|
|
# Only use monolith if we are set to
|
|
matchrdirpath = (
|
|
next((match for match in matches if match.value == self._resourcedir), None)
|
|
if self.directory_load
|
|
else None
|
|
)
|
|
if not matchrdirpath and crawl:
|
|
for match in matches:
|
|
if path == "/rest/v1" and not loadcomplete:
|
|
if (
|
|
str(match.full_path) == "links.Schemas.href"
|
|
or str(match.full_path) == "links.Registries.href"
|
|
):
|
|
continue
|
|
elif not loadcomplete:
|
|
if (
|
|
str(match.full_path) == "Registries.@odata.id"
|
|
or str(match.full_path) == "JsonSchemas.@odata.id"
|
|
):
|
|
continue
|
|
|
|
if match.value == path:
|
|
continue
|
|
elif not isinstance(match.value, six.string_types):
|
|
continue
|
|
|
|
href = "%s" % match.value
|
|
self._load(
|
|
href,
|
|
crawl=crawl,
|
|
originaluri=originaluri,
|
|
includelogs=includelogs,
|
|
init=init,
|
|
prevpath=fpath(str(match.full_path), path),
|
|
loadcomplete=loadcomplete,
|
|
)
|
|
elif crawl:
|
|
href = "%s" % matchrdirpath.value
|
|
self._load(
|
|
href,
|
|
crawl=crawl,
|
|
originaluri=originaluri,
|
|
includelogs=includelogs,
|
|
init=init,
|
|
prevpath=path,
|
|
loadcomplete=loadcomplete,
|
|
)
|
|
if loadcomplete:
|
|
if path == "/rest/v1":
|
|
schemamatch = jsonpath_rw.parse("$..extref")
|
|
else:
|
|
schemamatch = jsonpath_rw.parse("$..Uri")
|
|
smatches = schemamatch.find(resp.dict)
|
|
matches = matches + smatches
|
|
for match in matches:
|
|
if isinstance(match.value, six.string_types):
|
|
self._load(
|
|
match.value,
|
|
crawl=crawl,
|
|
originaluri=originaluri,
|
|
includelogs=includelogs,
|
|
init=init,
|
|
loadcomplete=loadcomplete,
|
|
prevpath=fpath(str(match.full_path), path),
|
|
)
|
|
|
|
def _parse_schema(self, resp):
|
|
"""Function to get and replace schema $ref with data
|
|
|
|
:param resp: response data containing ref items.
|
|
:type resp: str
|
|
"""
|
|
# pylint: disable=maybe-no-member
|
|
if not self.typepath.gencompany:
|
|
return self._parse_schema_gen(resp)
|
|
jsonpath_expr = jsonpath_rw.parse('$.."$ref"')
|
|
matches = jsonpath_expr.find(resp.dict)
|
|
respcopy = resp.dict
|
|
typeregex = r"([#,@].*?\.)"
|
|
if matches:
|
|
for match in matches:
|
|
fullpath = str(match.full_path)
|
|
jsonfile = match.value.split("#")[0]
|
|
jsonpath = match.value.split("#")[1]
|
|
listmatch = None
|
|
found = None
|
|
|
|
if "redfish.dmtf.org" in jsonfile:
|
|
if "odata" in jsonfile:
|
|
jsonpath = jsonpath.replace(jsonpath.split("/")[-1], "odata" + jsonpath.split("/")[-1])
|
|
jsonfile = "Resource.json"
|
|
|
|
found = re.search(typeregex, fullpath)
|
|
if found:
|
|
repitem = fullpath[found.regs[0][0] : found.regs[0][1]]
|
|
schemapath = "/" + fullpath.replace(repitem, "~").replace(".", "/").replace("~", repitem)
|
|
else:
|
|
schemapath = "/" + fullpath.replace(".", "/")
|
|
|
|
if ".json" in jsonfile:
|
|
itempath = schemapath
|
|
|
|
if self.is_redfish:
|
|
if resp.request.path[-1] == "/":
|
|
newpath = "/".join(resp.request.path.split("/")[:-2]) + "/" + jsonfile + "/"
|
|
else:
|
|
newpath = "/".join(resp.request.path.split("/")[:-1]) + "/" + jsonfile + "/"
|
|
else:
|
|
newpath = "/".join(resp.request.path.split("/")[:-1]) + "/" + jsonfile
|
|
|
|
if "href.json" in newpath:
|
|
continue
|
|
|
|
if newpath.lower() not in self.visited_urls:
|
|
self.load(
|
|
newpath,
|
|
crawl=False,
|
|
includelogs=False,
|
|
init=False,
|
|
loadtype="ref",
|
|
)
|
|
|
|
instance = list()
|
|
|
|
# deprecated type "string" for Type.json
|
|
if "string" in self.types:
|
|
for item in self.iter("string"):
|
|
instance.append(item)
|
|
if "object" in self.types:
|
|
for item in self.iter("object"):
|
|
instance.append(item)
|
|
|
|
for item in instance:
|
|
if jsonfile in item.path:
|
|
if "anyOf" in fullpath:
|
|
break
|
|
|
|
dictcopy = item.dict
|
|
try:
|
|
# TODO may need to really verify this is acceptable regex
|
|
listmatch = re.search("[][0-9]+[]", itempath)
|
|
except regexerr:
|
|
pass
|
|
# LOGGER.info("An error occurred with regex match on path: %s\n%s\n"\
|
|
# % (itempath, str(excp)))
|
|
|
|
if listmatch:
|
|
start = listmatch.regs[0][0]
|
|
end = listmatch.regs[0][1]
|
|
|
|
newitempath = [itempath[:start], itempath[end:]]
|
|
start = jsonpointer.JsonPointer(newitempath[0])
|
|
end = jsonpointer.JsonPointer(newitempath[1])
|
|
|
|
del start.parts[-1], end.parts[-1]
|
|
vals = start.resolve(respcopy)
|
|
|
|
count = 0
|
|
|
|
for val in vals:
|
|
try:
|
|
if "$ref" in six.iterkeys(end.resolve(val)):
|
|
end.resolve(val).pop("$ref")
|
|
end.resolve(val).update(dictcopy)
|
|
replace_pointer = jsonpointer.JsonPointer(end.path + jsonpath)
|
|
|
|
data = replace_pointer.resolve(val)
|
|
set_pointer(val, end.path, data)
|
|
start.resolve(respcopy)[count].update(val)
|
|
|
|
break
|
|
except:
|
|
count += 1
|
|
else:
|
|
itempath = jsonpointer.JsonPointer(itempath)
|
|
del itempath.parts[-1]
|
|
|
|
try:
|
|
if "$ref" in six.iterkeys(itempath.resolve(respcopy)):
|
|
itempath.resolve(respcopy).pop("$ref")
|
|
itempath.resolve(respcopy).update(dictcopy)
|
|
break
|
|
except jsonpointer.JsonPointerException:
|
|
pass
|
|
|
|
if jsonpath:
|
|
if "anyOf" in fullpath:
|
|
continue
|
|
|
|
if not jsonfile:
|
|
replacepath = jsonpointer.JsonPointer(jsonpath)
|
|
schemapath = schemapath.replace("/$ref", "")
|
|
if re.search(r"\[\d]", schemapath):
|
|
schemapath = schemapath.translate(str.maketrans("", "", "[]"))
|
|
schemapath = jsonpointer.JsonPointer(schemapath)
|
|
data = replacepath.resolve(respcopy)
|
|
|
|
if "$ref" in schemapath.resolve(respcopy):
|
|
schemapath.resolve(respcopy).pop("$ref")
|
|
schemapath.resolve(respcopy).update(data)
|
|
|
|
else:
|
|
if not listmatch:
|
|
schemapath = schemapath.replace("/$ref", "")
|
|
replacepath = schemapath + jsonpath
|
|
replace_pointer = jsonpointer.JsonPointer(replacepath)
|
|
try:
|
|
data = replace_pointer.resolve(respcopy)
|
|
set_pointer(respcopy, schemapath, data)
|
|
except jsonpointer.JsonPointerException:
|
|
# TODO
|
|
pass
|
|
|
|
resp.loaddict(respcopy)
|
|
else:
|
|
resp.loaddict(respcopy)
|
|
|
|
def _parse_schema_gen(self, resp):
|
|
"""Redfish general function to get and replace schema $ref with data
|
|
|
|
:param resp: response data containing ref items.
|
|
:type resp: str
|
|
|
|
"""
|
|
# pylint: disable=maybe-no-member
|
|
getval = lambda inmat: getval(inmat.left) + "/" + str(inmat.right) if hasattr(inmat, "left") else str(inmat)
|
|
respcopy = resp.dict
|
|
jsonpath_expr = jsonpath_rw.parse('$.."anyOf"')
|
|
while True:
|
|
matches = jsonpath_expr.find(respcopy)
|
|
if not matches:
|
|
break
|
|
match = matches[0]
|
|
newval = None
|
|
schlist = match.value
|
|
schlist = [ele for ele in list(schlist) if ele != {"type": "null"}]
|
|
norefsch = [ele for ele in list(schlist) if isinstance(ele, dict) and len(ele.keys()) > 1]
|
|
if norefsch:
|
|
newval = norefsch[0]
|
|
else:
|
|
newsc = [ele for ele in list(schlist) if not ele["$ref"].split("#")[0]]
|
|
newval = newsc[0] if newsc else None
|
|
if not newval:
|
|
schlist = [
|
|
ele["$ref"]
|
|
for ele in list(schlist)
|
|
if "$ref" in ele.keys()
|
|
and (ele["$ref"].split("#")[0].endswith(".json") and "odata" not in ele["$ref"].split("#")[0])
|
|
]
|
|
maxsch = max(schlist)
|
|
newval = {"$ref": maxsch}
|
|
|
|
itempath = "/" + getval(match.full_path)
|
|
if re.search(r"\[\d+]", itempath):
|
|
itempath = itempath.translate(str.maketrans("", "", "[]"))
|
|
itempath = jsonpointer.JsonPointer(itempath)
|
|
del itempath.parts[-1]
|
|
if "anyOf" in six.iterkeys(itempath.resolve(respcopy)):
|
|
itempath.resolve(respcopy).pop("anyOf")
|
|
itempath.resolve(respcopy).update(newval)
|
|
|
|
jsonpath_expr = jsonpath_rw.parse('$.."$ref"')
|
|
matches = jsonpath_expr.find(respcopy)
|
|
if matches:
|
|
for _, match in enumerate(matches):
|
|
jsonfile = match.value.split("#")[0]
|
|
jsonfile = "" if jsonfile.lower() == resp.request.path.lower() else jsonfile
|
|
jsonpath = match.value.split("#")[1]
|
|
|
|
schemapath = "/" + getval(match.full_path)
|
|
if jsonfile:
|
|
itempath = schemapath
|
|
if "/" not in jsonfile:
|
|
inds = -2 if resp.request.path[-1] == "/" else -1
|
|
jsonfile = "/".join(resp.request.path.split("/")[:inds]) + "/" + jsonfile + "/"
|
|
if jsonfile not in self.paths:
|
|
self.load(
|
|
jsonfile,
|
|
crawl=False,
|
|
includelogs=False,
|
|
init=False,
|
|
loadtype="ref",
|
|
)
|
|
item = self.paths[jsonfile] if jsonfile in self.paths else None
|
|
|
|
if not item:
|
|
if "anyOf" not in schemapath:
|
|
raise SchemaValidationError()
|
|
continue
|
|
if re.search(r"\[\d+]", itempath):
|
|
itempath = itempath.translate(str.maketrans("", "", "[]"))
|
|
itempath = jsonpointer.JsonPointer(itempath)
|
|
del itempath.parts[-1]
|
|
try:
|
|
if "$ref" in six.iterkeys(itempath.resolve(respcopy)):
|
|
itempath.resolve(respcopy).pop("$ref")
|
|
itempath.resolve(respcopy).update(item.dict)
|
|
except jsonpointer.JsonPointerException:
|
|
pass
|
|
|
|
if jsonpath:
|
|
schemapath = schemapath.replace("/$ref", "")
|
|
if re.search(r"\[\d+]", schemapath):
|
|
schemapath = schemapath.translate(str.maketrans("", "", "[]"))
|
|
if not jsonfile:
|
|
replacepath = jsonpointer.JsonPointer(jsonpath)
|
|
schemapath = jsonpointer.JsonPointer(schemapath)
|
|
data = replacepath.resolve(respcopy)
|
|
if "$ref" in schemapath.resolve(respcopy):
|
|
schemapath.resolve(respcopy).pop("$ref")
|
|
schemapath.resolve(respcopy).update(data)
|
|
else:
|
|
replacepath = schemapath + jsonpath
|
|
replace_pointer = jsonpointer.JsonPointer(replacepath)
|
|
data = replace_pointer.resolve(respcopy)
|
|
set_pointer(respcopy, schemapath, data)
|
|
|
|
resp.loaddict(respcopy)
|
|
else:
|
|
resp.loaddict(respcopy)
|
|
|
|
def load_from_dict(self, src):
|
|
"""Load data to monolith from a dict. This is the reverse of :func:`to_dict`.
|
|
|
|
:param src: data receive from rest operation.
|
|
:type src: str
|
|
|
|
"""
|
|
self._type = src["Type"]
|
|
self._name = src["Name"]
|
|
self.typesadded = defaultdict(set, {ki: set(val) for ki, val in src["typepath"].items()})
|
|
self.ctree = defaultdict(set, {ki: set(val) for ki, val in src["ctree"].items()})
|
|
self.colltypes = defaultdict(set, {ki: set(val) for ki, val in src["colls"].items()})
|
|
for _, resp in list(src["resps"].items()):
|
|
member = RisMonolithMemberv100(None, self.is_redfish)
|
|
member.load_from_dict(resp)
|
|
self.update_member(member=member, init=False)
|
|
|
|
def to_dict(self):
|
|
"""Convert data to a dict from monolith. This is the reverse of :func:`load_from_dict`."""
|
|
result = OrderedDict()
|
|
result["Type"] = self.type
|
|
result["Name"] = self.name
|
|
result["typepath"] = self.typesadded
|
|
result["ctree"] = self.ctree
|
|
result["colls"] = self.colltypes
|
|
result["resps"] = {x: v.to_dict() for x, v in list(self.paths.items())}
|
|
return result
|
|
|
|
def markmodified(self, opath, path=None, modpaths=None):
|
|
"""Mark the paths to be modifed which are connected to current path. When calling this
|
|
function you only need to include `opath`.
|
|
|
|
:param opath: original path which has been modified
|
|
:type opath: str
|
|
"""
|
|
modpaths = set() if modpaths is None else modpaths
|
|
path = path if path else opath
|
|
if not path:
|
|
return
|
|
modpaths.update(self.ctree[path] if path in self.ctree else set())
|
|
self.paths[path].modified = True
|
|
for npath in [
|
|
unmodpath for unmodpath in modpaths if unmodpath in self.paths and not self.paths[unmodpath].modified
|
|
]:
|
|
self.markmodified(opath, path=npath, modpaths=modpaths)
|
|
return modpaths
|
|
|
|
def checkmodified(self, opath, path=None, modpaths=None):
|
|
"""Check if the path or its children are modified. When calling this
|
|
function you only need to include `opath`.
|
|
|
|
:param opath: original path which has been modified
|
|
:type opath: str
|
|
"""
|
|
# return [paths for paths in self.ctree[path] if self.paths[paths].modified]
|
|
modpaths = set() if modpaths is None else modpaths
|
|
path = path if path else opath
|
|
newpaths = set()
|
|
if not path:
|
|
return
|
|
if path in self.paths and self.paths[path].modified:
|
|
newpaths = (
|
|
set([conn for conn in self.ctree[path] if conn in self.paths and self.paths[path].modified]) - modpaths
|
|
)
|
|
modpaths.update(newpaths | set([path]))
|
|
for npath in [unmodpath for unmodpath in newpaths]:
|
|
self.checkmodified(opath, path=npath, modpaths=modpaths)
|
|
return modpaths
|
|
|
|
def removepath(self, path):
|
|
"""Remove a given path from the cache
|
|
|
|
:param path: path which is to be checked if modified
|
|
:type path: str
|
|
"""
|
|
if path in self._visited_urls:
|
|
self._visited_urls.remove(path)
|
|
if path not in self.paths:
|
|
return
|
|
if path in self.typesadded[self.paths[path].maj_type]:
|
|
self.typesadded[self.paths[path].maj_type].remove(path)
|
|
if not self.typesadded[self.paths[path].maj_type]:
|
|
del self.typesadded[self.paths[path].maj_type]
|
|
del self.paths[path]
|
|
if path in self.ctree:
|
|
del self.ctree[path]
|
|
_ = [self.ctree[paths].remove(path) for paths in self.ctree if path in self.ctree[paths]]
|
|
|
|
def _populatecollections(self):
|
|
"""Populate the collections type and types depending on resourcedirectory"""
|
|
if self._resourcedir not in self.paths:
|
|
return
|
|
self.colltypes = defaultdict(set)
|
|
alltypes = []
|
|
colls = []
|
|
for item in self.paths[self._resourcedir].dict["Instances"]:
|
|
# Fix for incorrect RDir instances.
|
|
if self.typepath.defs.typestring not in item or item[self.typepath.defs.hrefstring] in self.paths:
|
|
continue
|
|
typename = ".".join(item[self.typepath.defs.typestring].split(".", 2)[:2]).split("#")[-1]
|
|
_ = [alltypes.append(typename) if "Collection" not in typename else None]
|
|
_ = [colls.append(typename) if "Collection" in typename else None]
|
|
member = RisMonolithMemberv100(None, self.is_redfish)
|
|
member.popdefs(typename, item[self.typepath.defs.hrefstring], item[self.etagstr])
|
|
self.update_member(member=member, init=False)
|
|
for coll in colls:
|
|
collname = coll.split("Collection")[0].split("#")[-1]
|
|
typename = next((name for name in alltypes if name.startswith(collname)), None)
|
|
colltype = ".".join(coll.split(".", 2)[:2]).split("#")[-1]
|
|
self.colltypes[typename].add(colltype)
|
|
|
|
def capture(self, redmono=False, single=True):
|
|
"""Crawls the server and returns the monolith data or just headers and responses.
|
|
|
|
:param redmono: If True, returns only headers and responses; otherwise, returns full monolith data.
|
|
:type redmono: bool
|
|
:rtype: dict
|
|
"""
|
|
self.load(includelogs=False, crawl=True, loadcomplete=True, path_refresh=True, init=True, single=single)
|
|
|
|
if not redmono:
|
|
return self.to_dict()
|
|
|
|
ret = {}
|
|
try:
|
|
for x, v in self.paths.items():
|
|
if v:
|
|
v_resp = v.resp
|
|
if v_resp:
|
|
ret[x] = {"Headers": v_resp.getheaders(), "Response": v_resp.dict}
|
|
except Exception as e:
|
|
LOGGER.debug("Error in capture: %s", e)
|
|
ret = {}
|
|
return ret
|
|
|
|
def normalize_urls(self, data):
|
|
"""
|
|
Normalize URL-like dictionary keys:
|
|
- If both '/path' and '/path/' exist, keep only '/path/'.
|
|
- If only one version exists, ensure the key ends with '/'.
|
|
- All final keys end with '/'.
|
|
"""
|
|
normalized_data = {}
|
|
seen = set()
|
|
|
|
for key in data:
|
|
# Normalize key with trailing slash
|
|
if key.endswith("/"):
|
|
base_key = key[:-1]
|
|
trailing_key = key
|
|
else:
|
|
base_key = key
|
|
trailing_key = key + "/"
|
|
|
|
# Avoid processing the same base path twice
|
|
if base_key in seen:
|
|
continue
|
|
seen.add(base_key)
|
|
|
|
# Prefer version with trailing slash
|
|
if trailing_key in data:
|
|
normalized_data[trailing_key] = data[trailing_key]
|
|
elif base_key in data:
|
|
normalized_data[trailing_key] = data[base_key]
|
|
|
|
return normalized_data
|
|
|
|
def fetch_data(self, path):
|
|
resp = self.client.get(path)
|
|
return resp
|
|
|
|
def captureallconfig(self, single=True, exclueurl=None):
|
|
"""Crawls the server and returns the monolith data or just headers and responses.
|
|
|
|
:param redmono: If True, returns only headers and responses; otherwise, returns full monolith data.
|
|
:type redmono: bool
|
|
:rtype: dict
|
|
"""
|
|
self.loadallconfig(
|
|
includelogs=False, crawl=True, loadcomplete=True, path_refresh=True, init=True, single=single
|
|
)
|
|
|
|
ret = {}
|
|
|
|
try:
|
|
for x, v in self.paths.items():
|
|
if exclueurl.lower() != x.lower():
|
|
if v:
|
|
if v.resp is not None:
|
|
ret[x] = {"Headers": v.resp.getheaders(), "Response": v.resp.dict}
|
|
else:
|
|
resp = self.fetch_data(x)
|
|
ret[x] = {"Headers": resp.getheaders(), "Response": resp.dict}
|
|
except Exception as e:
|
|
LOGGER.debug("Error in capture: %s", e)
|
|
ret = {}
|
|
ret = self.normalize_urls(ret)
|
|
return ret
|
|
|
|
def killthreads(self):
|
|
"""Function to kill threads on logout"""
|
|
threads = []
|
|
for thread in threading.enumerate():
|
|
if isinstance(thread, LoadWorker):
|
|
self.get_queue.put(
|
|
(
|
|
"KILL",
|
|
"KILL",
|
|
"KILL",
|
|
"KILL",
|
|
"KILL",
|
|
"KILL",
|
|
"KILL",
|
|
"KILL",
|
|
"KILL",
|
|
"KILL",
|
|
)
|
|
)
|
|
threads.append(thread)
|
|
|
|
for thread in threads:
|
|
thread.join()
|
|
|
|
def _update_progress(self):
|
|
"""Simple function to increment the dot progress"""
|
|
if self.progress % 6 == 0:
|
|
sys.stdout.write(".")
|