azure-sdk-for-python: privatelink endpoints not working

from azure.storage.filedatalake import DataLakeServiceClient

account_url = "https://[ommitted].privatelink.dfs.core.windows.net/"
key1 = "[ommitted]"
fs_name = "[ommitted]"

service = DataLakeServiceClient(
    account_url=account_url,
    credential=key1)
fs = service.create_file_system(file_system=fs_name)

---------------------------------------------------------------------------
ServiceRequestError                       Traceback (most recent call last)
<ipython-input-54-68aacab587c5> in <module>
      2     account_url=account_url,
      3     credential=key1)
----> 4 directory = service.create_file_system(file_system=fs_name)

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\storage\filedatalake\_data_lake_service_client.py in create_file_system(self, file_system, metadata, public_access, **kwargs)
    248         """
    249         file_system_client = self.get_file_system_client(file_system)
--> 250         file_system_client.create_file_system(metadata=metadata, public_access=public_access, **kwargs)
    251         return file_system_client
    252 

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\storage\filedatalake\_file_system_client.py in create_file_system(self, metadata, public_access, **kwargs)
    244                 :caption: Creating a file system in the datalake service.
    245         """
--> 246         return self._container_client.create_container(metadata=metadata,
    247                                                        public_access=public_access,
    248                                                        **kwargs)

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\tracing\decorator.py in wrapper_use_tracer(*args, **kwargs)
     81             span_impl_type = settings.tracing_implementation()
     82             if span_impl_type is None:
---> 83                 return func(*args, **kwargs)
     84 
     85             # Merge span is parameter is set, but only if no explicit parent are passed

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\storage\blob\_container_client.py in create_container(self, metadata, public_access, **kwargs)
    284         container_cpk_scope_info = get_container_cpk_scope_info(kwargs)
    285         try:
--> 286             return self._client.container.create( # type: ignore
    287                 timeout=timeout,
    288                 access=public_access,

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\storage\blob\_generated\operations\_container_operations.py in create(self, timeout, metadata, access, request_id_parameter, container_cpk_scope_info, **kwargs)
    126 
    127         request = self._client.put(url, query_parameters, header_parameters)
--> 128         pipeline_response = self._client._pipeline.run(request, stream=False, **kwargs)
    129         response = pipeline_response.http_response
    130 

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in run(self, request, **kwargs)
    209             else _TransportRunner(self._transport)
    210         )
--> 211         return first_node.send(pipeline_request)  # type: ignore

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\policies\_redirect.py in send(self, request)
    156         redirect_settings = self.configure_redirects(request.context.options)
    157         while retryable:
--> 158             response = self.next.send(request)
    159             redirect_location = self.get_redirect_location(response)
    160             if redirect_location and redirect_settings['allow']:

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\storage\filedatalake\_shared\policies.py in send(self, request)
    513                     self.sleep(retry_settings, request.context.transport)
    514                     continue
--> 515                 raise err
    516         if retry_settings['history']:
    517             response.context['history'] = retry_settings['history']

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\storage\filedatalake\_shared\policies.py in send(self, request)
    487         while retries_remaining:
    488             try:
--> 489                 response = self.next.send(request)
    490                 if is_retry(response, retry_settings['mode']):
    491                     retries_remaining = self.increment(

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\storage\filedatalake\_shared\policies.py in send(self, request)
    288             request.context.options.pop('raw_response_hook', self._response_callback)
    289 
--> 290         response = self.next.send(request)
    291         will_retry = is_retry(response, request.context.options.get('mode'))
    292         if not will_retry and download_stream_current is not None:

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
     69         _await_result(self._policy.on_request, request)
     70         try:
---> 71             response = self.next.send(request)
     72         except Exception:  # pylint: disable=broad-except
     73             if not _await_result(self._policy.on_exception, request):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\_base.py in send(self, request)
    101         return PipelineResponse(
    102             request.http_request,
--> 103             self._sender.send(request.http_request, **request.context.options),
    104             context=request.context,
    105         )

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\storage\filedatalake\_shared\base_client.py in send(self, request, **kwargs)
    317 
    318     def send(self, request, **kwargs):
--> 319         return self._transport.send(request, **kwargs)
    320 
    321     def open(self):

~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\azure\core\pipeline\transport\_requests_basic.py in send(self, request, **kwargs)
    283 
    284         if error:
--> 285             raise error
    286         return RequestsTransportResponse(request, response, self.connection_config.data_block_size)

ServiceRequestError: ("hostname '[ommitted].privatelink.blob.core.windows.net' doesn't match either of '*.blob.core.windows.net', '*.mwh01prdstrz01a.store.core.windows.net', '*.blob.storage.azure.net', '*.z1.blob.storage.azure.net', '*.z2.blob.storage.azure.net', '*.z3.blob.storage.azure.net', '*.z4.blob.storage.azure.net', '*.z5.blob.storage.azure.net', '*.z6.blob.storage.azure.net', '*.z7.blob.storage.azure.net', '*.z8.blob.storage.azure.net', '*.z9.blob.storage.azure.net', '*.z10.blob.storage.azure.net', '*.z11.blob.storage.azure.net', '*.z12.blob.storage.azure.net', '*.z13.blob.storage.azure.net', '*.z14.blob.storage.azure.net', '*.z15.blob.storage.azure.net', '*.z16.blob.storage.azure.net', '*.z17.blob.storage.azure.net', '*.z18.blob.storage.azure.net', '*.z19.blob.storage.azure.net', '*.z20.blob.storage.azure.net', '*.z21.blob.storage.azure.net', '*.z22.blob.storage.azure.net', '*.z23.blob.storage.azure.net', '*.z24.blob.storage.azure.net', '*.z25.blob.storage.azure.net', '*.z26.blob.storage.azure.net', '*.z27.blob.storage.azure.net', '*.z28.blob.storage.azure.net', '*.z29.blob.storage.azure.net', '*.z30.blob.storage.azure.net', '*.z31.blob.storage.azure.net', '*.z32.blob.storage.azure.net', '*.z33.blob.storage.azure.net', '*.z34.blob.storage.azure.net', '*.z35.blob.storage.azure.net', '*.z36.blob.storage.azure.net', '*.z37.blob.storage.azure.net', '*.z38.blob.storage.azure.net', '*.z39.blob.storage.azure.net', '*.z40.blob.storage.azure.net', '*.z41.blob.storage.azure.net', '*.z42.blob.storage.azure.net', '*.z43.blob.storage.azure.net', '*.z44.blob.storage.azure.net', '*.z45.blob.storage.azure.net', '*.z46.blob.storage.azure.net', '*.z47.blob.storage.azure.net', '*.z48.blob.storage.azure.net', '*.z49.blob.storage.azure.net', '*.z50.blob.storage.azure.net'",)

About this issue

  • Original URL
  • State: closed
  • Created 3 years ago
  • Reactions: 1
  • Comments: 19 (9 by maintainers)

Most upvoted comments

@isgallagher This github issue has been open for quite sometime. I wanted to do a quick follow-up to check if you need any further assistance on this.

I ended up adding the public endpoints into the DNS to resolve to the private IPs. Not great but it works. I really would like to be able to specify a custom URL for the endpoint like with the AWS CLI / boto3.

I’ve responded to the email offline @isgallagher

@isgallagher if it helps the URL for the storage automatically CNAMEs to the private link address on vm. I got around this by using the storage account’s URL instead of the private link one in my connection.

Thanks for the feedback, we’ll investigate asap.