­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ 3 cj@sddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z mZmZmZmZddlmZmZmZmZmZddlmZddlmZddlmZdd lmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'ddl(m)Z)dZ*dZ+dZ,dZ-e*e+e,gZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6ej7e8Z9dddfdddfd d!dfd"Z:e Gd#d$d$e Z;Gd%d&d&e<Z=Gd'd(d(e<Z>Gd)d*d*e<Z?d+ffd,d-Z@e3fd.d/ZAed0d1d2d3d4gZBed5d6d7gZCGd8dde'ejDd9ZEd:d;ZFeeEeGfd<d=d>ZHd?d@ZIdVeGeJdBdCdDZKdEdFZLdWdHdIZMGdJdKdKeNZOdLdMZPeEeGeJdNdOdPZQeGeeEdQdRdSZReGd<dTdUZSdS)XN) namedtuple)Enumunique)AnyDictListOptionalTuple) atomic_helperdmiimporternet type_utils) user_data)util) write_json)Distro) EventScope EventType) launch_index)Paths)CloudInitPickleMixin)eventsZdisabledlocalr passZ FILESYSTEMNETWORK DataSourcez|EXPERIMENTAL: The structure and format of content scoped under the 'ds' key may change in subsequent releases of cloud-init.zredacted for non-root userz cloud-nameZ_unsetunknownz aws-chinacCs|dkS)Naws)crr/usr/lib/python3.6/__init__.py?sr"zaws-govcCs|dkS)Nrr)r rrr!r"@sz azure-chinacCs|dkS)NZazurer)r rrr!r"As)zcn-zus-gov-Zchinac@s2eZdZdZdZdZdZdZdZe ddd Z d S) NetworkConfigSourcezb Represents the canonical list of network config sources that cloud-init knows about. cmdlinedsZ system_cfgZfallbackZ initramfs)returncCs|jS)N)value)selfrrr!__str__RszNetworkConfigSource.__str__N) __name__ __module__ __qualname____doc__CMD_LINEDS SYSTEM_CFGZFALLBACK INITRAMFSstrr)rrrr!r#Esr#c@seZdZdZdS)DatasourceUnpickleUserDataErrorzERaised when userdata is unable to be unpickled due to python upgradesN)r*r+r,r-rrrr!r3Vsr3c@s eZdZdS)DataSourceNotFoundExceptionN)r*r+r,rrrr!r4Zsr4c@seZdZdZdS)InvalidMetaDataExceptionz8Raised when metadata is broken, unavailable or disabled.N)r*r+r,r-rrrr!r5^sr5c Cstj|}g}g}x|jD]\}}|r6|d|}n|}|j|ksR|j|kr\|j|t|tr|jdr|j||jdd||<t|t rt |||} |j | j d|j | j d| ||<qWt ||d<t ||d<|S)zProcess all instance metadata cleaning it up for persisting as json. Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list @return Dict copy of processed metadata. /zci-b64:r6base64_encoded_keyssensitive_keys)copydeepcopyitemslowerappend isinstancer2 startswithreplacedictprocess_instance_metadataextendpopsorted) metadatakey_pathr9md_copyr8Z sens_keyskeyvalZ sub_key_pathZ return_valrrr!rCbs,          rCcCs|jdgs|Stj|}xh|jdD]Z}|jd}|}x6|D].}||kr>t||tr>||dkr>||}q>W||kr&|||<q&W|S)zRedact any sensitive keys from to provided metadata dictionary. Replace any keys values listed in 'sensitive_keys' with redact_value. r9r7)getr:r;splitr?rB)rGZ redact_valuerIrH path_partsobjpathrrr!redact_sensitive_keyss       rS URLParamsZmax_wait_secondsZtimeout_secondsZ num_retriessec_between_retriesDataSourceHostnamehostname is_defaultc @seZdZUeZdZdZdZdZdZ dZ e j e j e je jfZee dfd|ZdZdZdZejejejejejhiZejejhiZdefd efd ifd}d~ddddf Z eee!e"fdf dZ#dZ$ee!df$dZ%dZ&de'e(dddZ)e*ddddZ+d d!Z,e-d"d#d$Z.e-d"d%d&Z/d'd(Z0d)d*Z1ffd+d,Z2e-d"d-d.Z3dd0d1Z4e-d"d2d3Z5d4d5Z6dd6d7Z7d8d9Z8d:d;Z9e:dd?Ze:dDdEZ?dFdGZ@e:dHdIZAdJdKZBe:dLdMZCdNdOZDdPdQZEdRdSZFdTdUZGdVdWZHdXdYZIdZd[ZJd\d]ZKd^d_ZLe:d`daZMe:dbdcZNdddeZOddfdgZPdhdiZQeRedjdkdlZSeRee-dmdndoZTdpdqZUdrdsZVeWddtduZXe:dvdwZYdxdyZZdzd{Z[dS)rz en_US.UTF-8Z_undefN.rL  ec2_metadata network_jsonrGuserdata userdata_raw vendordatavendordata_raw vendordata2vendordata2_rawFcombined_cloud_config merged_cfgmerged_system_cfgsecurity-credentials user-datar vendor-datads/vendor_data)distropathscCs~||_||_||_d|_i|_d|_d|_d|_d|_d|_ t j |jd|j fi|_ |j s`i|_ |sttj|j|_n||_dS)NZ datasource)sys_cfgrjrkr]rGr^r_rar`rbrZget_cfg_by_pathdsnameds_cfgudZUserDataProcessorud_proc)r(rlrjrkrprrr!__init__s"zDataSource.__init__)ci_pkl_versionr&cCst|dsd|_t|ds d|_t|ds0d|_t|dsJt|dddt|d r|jdk ryt|jWn6tk r}ztj d |t |WYdd}~XnXdS) z(Perform deserialization fixes for Paths.raNrbskip_hotplug_detectFcheck_if_fallback_is_allowedcSsdS)NFrrrrr!r"=sz&DataSource._unpickle..r]z:Unable to unpickle datasource: %s. Ignoring current cache.) hasattrrarbrssetattrr]r2AttributeErrorLOGdebugr3)r(rrerrr! _unpickle3s     zDataSource._unpicklecCs tj|S)N)robj_name)r(rrr!r)OszDataSource.__str__)r&cCsdS)z#Check if running on this datasourceTr)r(rrr! ds_detectRszDataSource.ds_detectcCsN|jjtjkr$tjd|dS|jjdg|jgkrJtjd|dSdS)aIOverride if either: - only a single datasource defined (nothing to fall back to) - commandline argument is used (ci.ds=OpenStack) Note: get_cmdline() is required for the general case - when ds-identify does not run, _something_ needs to detect the kernel command line definition. zOMachine is configured by the kernel commandline to run on single datasource %s.TZdatasource_listz5Machine is configured to run on single datasource %s.F)rmr= parse_cmdlinerxryrlrN)r(rrr!override_ds_detectVs zDataSource.override_ds_detectcCs@|jr|jS|jr,tjd||jStjd|dSdS)z&Overrides runtime datasource detectionz8Detected platform: %s. Checking for active instance dataz#Datasource type %s is not detected.FN)r _get_datar}rxry)r(rrr!_check_and_get_datams zDataSource._check_and_get_datacCs|jj}|j}|j}|d}ddg||t|j|j|j|j|j|dd|dd|dd|j|j|d|||d d|||d d |j|j |d |d d iS)z2Return a dictionary of standardized metadata keys.sys_infov1 subplatformZdistrrLpythonunameplatformvariant)Z _beta_keyszavailability-zoneavailability_zonecloud_idz cloud-name cloud_namerjZdistro_versionZdistro_releaserZpublic_ssh_keysZpython_versionz instance-id instance_idZkernel_releasezlocal-hostnamelocal_hostnamemachineregionrZsystem_platformr) get_hostnamerWget_instance_idrcanonical_cloud_idrr platform_typeget_public_ssh_keysr)r( instance_datarrrZsysinforrr!_get_standardized_metadata{s8      z%DataSource._get_standardized_metadatacCsP|js dS|r|}n|j}x&|D]\}}t||r t|||q W|sLd|_dS)zReset any cached metadata attributes to datasource defaults. @param attr_defaults: Optional tuple of (attr, value) pairs to set instead of cached_attr_defaults. NF) _dirty_cachecached_attr_defaultsrurv)r(Z attr_defaultsZ attr_valuesZ attributer'rrr!clear_cached_attrss zDataSource.clear_cached_attrscCs"d|_|j}|s|S|j|S)zDatasources implement _get_data to setup metadata and userdata_raw. Minimally, the datasource should return a boolean True on success. T)rrpersist_instance_data)r(Z return_valuerrr!get_datas zDataSource.get_dataTcCst|r&tjj|jjr&t||jjdt|drbtj t |d}|j dd|j ddd|i}n^dd|j ii}t|drt |d}|t kr||dd<t|d rt |d }|t kr||dd <t|dd <tj |j|d <d |d d <tj |d |d <d|d d <tj|d<|j|j|y"tj|}ttj||jd}Wnbtk rv}ztjdt|dSd}~Xn2tk r}ztjdt|dSd}~XnX|jjd} |dj dd} tjj!|jj"d} tj#| d| | dd} | d| } tjj$| rtjj%| } tj&| | dd| rH| | krHtj'| t(| |dd|jjd}t(|t)|dS)aPProcess and write INSTANCE_JSON_FILE with all instance metadata. Replace any hyphens with underscores in key names for use in template processing. :param write_cache: boolean set True to persist obj.pkl when instance_link exists. @return True on successful write, False otherwise. Zobj_pklZ_crawled_metadataz user-dataNz vendor-datar%Z meta_datar\r[Z_docrdz, (str, bool). is_default is a bool and it's true only if hostname is localhost and was returned by util.get_hostname() as a default. This is used to differentiate with a user-defined localhost hostname. Optionally return (None, False) when metadata_only is True and local-hostname data is not available. Z localdomainZ localhostFzlocal-hostnameNT.rzip-%srrLz%s.%s)rGrNrVrrZget_fqdn_from_hostsfindr2rOr Zis_ipv4_addressZ gethostbyaddrrArr) r(ZfqdnZ resolve_ipZ metadata_onlyZ defdomainZdefhostZdomainrXZtoksrWZ hosts_fqdnZlhostrrr!rsF         zDataSource.get_hostnamecCs|jj|dS)N)Z data_source)rjget_package_mirror_info)r(rrr!r_sz"DataSource.get_package_mirror_info)source_event_typescCsXi}xN|D]F}x@|jjD]2\}}||kr|j|s>t||<||j|qWq W|S)N)supported_update_eventsr<rNsetadd)r(rsupported_eventseventZ update_scopeZ update_eventsrrr!get_supported_eventsbs   zDataSource.get_supported_events)rr&cCs|j|}xH|jD]<\}}tjd|jdjdd|D|jd|tffqW|rp|j|j}|rpdStjd|djdd|Dd S) aRefresh cached metadata if the datasource supports this event. The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @param source_event_types: List of EventTypes which may trigger a metadata update. @return True if the datasource did successfully update cached metadata due to source_event_type. z:Update datasource metadata and %s config due to events: %sz, cSsg|] }|jqSr)r').0rrrr! sz;DataSource.update_metadata_if_supported..z _%s_configTz(Datasource %s not updated for events: %scSsg|] }|jqSr)r')rrrrr!rsF) rr<rxryr'rrrr)r(rrZscopeZmatched_eventsresultrrr!update_metadata_if_supportedos" z'DataSource.update_metadata_if_supportedcCsdS)NFr)r(rlrrr!check_instance_idszDataSource.check_instance_idcCsdS)acheck_if_fallback_is_allowed() Checks if a cached ds is allowed to be restored when no valid ds is found in local mode by checking instance-id and searching valid data through ds list. @return True if a ds allows fallback, False otherwise. Fr)r(rrr!rtsz'DataSource.check_if_fallback_is_allowedcCsR|dkr t}|dkrt}x4|D],}|dkr,q||kr8|Stjd|||SqW|S)Nz%invalid dsmode '%s', using default=%s)DSMODE_NETWORK VALID_DSMODESrxr)Z candidatesdefaultZvalid candidaterrr!_determine_dsmodes  zDataSource._determine_dsmodecCsdS)Nr)r(rrr!network_configszDataSource.network_configcCsdS)a(setup(is_new_instance) This is called before user-data and vendor-data have been processed. Unless the datasource has set mode to 'local', then networking per 'fallback' or per 'network_config' will have been written and brought up the OS at this point. Nr)r(is_new_instancerrr!setups zDataSource.setupcCsdS)aactivate(cfg, is_new_instance) This is called before the init_modules will be called but after the user-data and vendor-data have been fully processed. The cfg is fully up to date config, it contains a merged view of system config, datasource config, user config, vendor config. It should be used rather than the sys_cfg passed to __init__. is_new_instance is a boolean indicating if this is a new instance. Nr)r(Zcfgrrrr!activates zDataSource.activaterM)r]N)r^N)r_N)r`N)raN)rbN) rcrdrerfr]rgrr_rhri)N)T)F)FFF)NN)\r*r+r,rZdsmoderrmrrrrr#r.r1r0r/Znetwork_config_sourcesr rrrrrrrBOOT_NEW_INSTANCEZBOOTZ BOOT_LEGACYZHOTPLUGrZdefault_update_eventsrrr2rrrrsZ_ci_pkl_versionrrrqrr{r)boolr}rrrrrrrrrrrpropertyrrrrrrrrrrrrrrrrrrrrrrrrrrrrt staticmethodrrrrrrrr!rs   &  R<        H$    ) metaclasscCsg}|s |St|tr|jSt|ttfr4t|St|trxN|jD]B\}}t|tr`|g}t|ttfrHx|D]}|rt|j|qtWqHW|S)N)r?r2 splitlineslistrrBr<r>)Z pubkey_datakeysZ_keynameZklistZpkeyrrr!rs     r)r&cCst|||}dd|D}t|kr&dnd} tjd| |xt||D]\} } tjd| jddd | | fd | | f|d } yV| Htjd | | |||} | jt j grd | | f| _ | t j | fSWdQRXWqDtk rtjtd| YqDXqDWddj|}t|dS)NcSsg|]}tj|qSr)rr|)rrrrr!rszfind_source..Znetworkrz#Searching for %s data source in: %sz search-%srr6zsearching for %s data from %szno %s data found from %s)name descriptionmessageparentz%Seeing if we can get any data from %szfound %s data from %szGetting data from %s failedz4Did not find any data source, searched classes: (%s)z, ) list_sources DEP_NETWORKrxryziprZReportEventStackrArrrrrr|rrrrr4)rlrjrkZds_depscfg_listpkg_listZreporterds_listZds_namesrrclsZmyrepsmsgrrr! find_sources.      rc Csg}tjd|||xt|D]l}tj|}tj||dg\}}|sLtjd|x8|D]0}tj|} t| d} | |} | rR|j| PqRWqW|S)zReturn a list of classes that have the same depends as 'depends' iterate through cfg_list, loading "DataSource*" modules and calling their "get_datasource_list". Return an ordered list of classes that match (if any) zLLooking for data source in: %s, via packages %s that matches dependencies %sZget_datasource_listzDCould not import %s. Does the DataSource exist and is it importable?) rxryr Z"match_case_insensitive_module_name find_moduleerror import_modulerrD) r dependsr Zsrc_listr%Zds_nameZm_locsZ _looked_locsZm_locmodZlisterZmatchesrrr!rs,       r system-uuid)fieldr&cCs*|sdStj|}|sdS|j|jkS)NF)r Z read_dmi_datar=)rrZ dmi_valuerrr!instance_id_matches_system_uuid9s  rcCsl|st}|st}|tkr(|tkr$|S|Sx2tjD]&\}}|\}}|j|r2||r2|Sq2W|tkrh|S|S)z@Lookup the canonical cloud-id for a given cloud_name and region.)rCLOUD_ID_REGION_PREFIX_MAPr<r@)rrrprefixZ cloud_id_testrZ valid_cloudrrr!rGsrTcCsj|sdSt|tr|St|tr*tj|St|trV|dkrNt|jdddStdtdt |dS)aLdata: a loaded object (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: None: return None string: return string list: return data the list is then processed in UserDataProcessor dict: return convert_vendordata(data.get('cloud-init')) NTz cloud-initF)recursez'vendordata['cloud-init'] cannot be dictz$Unknown data type for vendordata: %s) r?r2rr:r;rBconvert_vendordatarNrr)datarrrr!rZs     rc@s eZdZdS)BrokenMetadataN)r*r+r,rrrr!rrsrcCs8g}t|}x&|D]\}}|t|kr|j|qW|S)N)rr>)rr Zret_listZdepsetr Zdepsrrr!list_from_depends}s  r)rQfnamer&cCspytj|}Wn"tk r0tjtd|dSXytj||dddWn"tk rjtjtd|dSXdS)z[Use pickle to serialize Datasource to a file as a cache. :return: True on success zFailed pickling datasource %sFwb)Zomoderz Failed pickling datasource to %sT)pickledumpsrrrrxr)rQr Z pk_contentsrrr!rsr)r r&cCsd}ytj|dd}Wn<tk rR}z tjj|rBtjd||WYdd}~XnX|s\dSy tj |St k rzdStk rtj td|dSXdS)zBUse pickle to deserialize a instance Datasource from a cache file.NF)decodezfailed loading pickle in %s: %sz#Failed loading pickled blob from %s) rZ load_filerrrRisfilerxrr#rr3r)r Zpickle_contentsrzrrr!pkl_loads   r'cCstj}tjd|}tjd|}tjd|}|p6|p6|}|p>|}|rv|jdj}tjd|d|dd|d d |r|jdr|jdSd S) zCheck if command line argument for this datasource was passed Passing by command line overrides runtime datasource detection z ds=([^\s;]+)zci\.ds=([^\s;]+)zci\.datasource=([^\s;]+)rLz7Defining the datasource on the commandline using ci.ds=z or ci.datasource=z23.2zUse ds=z instead) deprecatedZdeprecated_versionZ extra_messager6)rZ get_cmdlineresearchgroupstripZ deprecate)r$Z ds_parse_0Z ds_parse_1Z ds_parse_2r%r(rmrrr!r~s     r~)r)T)Tabcr:rZloggingrr#r) collectionsrenumrrZtypingrrrrr Z cloudinitr r r r rrrorZcloudinit.atomic_helperrZcloudinit.distrosrZcloudinit.eventrrZcloudinit.filtersrZcloudinit.helpersrZcloudinit.persistencerZcloudinit.reportingrZDSMODE_DISABLEDZ DSMODE_LOCALrZ DSMODE_PASSrZDEP_FILESYSTEMrZ DS_PREFIXrZREDACT_SENSITIVE_VALUErrrZ getLoggerr*rxrr#rr3r4r5rCrSrTrVABCMetarrr2rrrrrrIOErrorrrrr'r~rrrr! s              #  +%