Extract signposts from jsonld (#1)

* use rdflib to extract signposts from jsonld

* use rdflib to extract signposts from jsonld

* linting

* unit tests for jsonld middleware

* update readme

* update flake pipeline

---------

Co-authored-by: Daniel Bauer <daniel.bauer@senckenberg.de>

.github/workflows/python-package.yml

@@ -32,7 +32,7 @@ jobs:
         python -m pip install -e '.[dev]'
     - name: Lint with flake8
       run: |
-        flake8 . --count --max-complexity=10 --max-line-length=127 --statistics
+        flake8 . --count --max-line-length=127 --statistics
     - name: Test with pytest
       run: |
         pytest

README.md

@@ -1,6 +1,6 @@
 [![Python package](https://github.com/dnlbauer/django-signposting/actions/workflows/python-package.yml/badge.svg)](https://github.com/dnlbauer/django-signposting/actions/workflows/python-package.yml)
 
-# FAIR signposting for Django
+# FAIR signposting middleware for Django
 
 `django_signposting` is a Django middleware library that facilitates the addition of
 FAIR signposting headers to HTTP responses.
@@ -11,6 +11,7 @@ Based on the [Signposting](https://github.com/stain/signposting) library.
 
 ## Features
 - Automatically adds signposting headers to HTTP responses.
+- Signposts can be added manually or automatically be parsed from JSON-LD/schema.org
 - Supports multiple relation types with optional media type specification.
 - Easily integrable with existing Django applications.
 
@@ -22,9 +23,30 @@ pip install django_signposting
 
 ## Usage
 
-### 1. Add Middleware
+### Automatic parsing of JSON-LD
 
-Add the middleware to your Django project's `MIDDLEWARE` setting in `settings.py`:
+To enable automatic parsing of JSON-LD, add the following middleware classes to your Django project's MIDDLEWARE setting in settings.py:
+
+```python
+MIDDLEWARE = [
+    ...,
+    'django_signposting.middleware.SignpostingMiddleware',
+    'django_signposting.middleware.JsonLdSignpostingParserMiddleware',
+    ...,
+]
+```
+
+This setup allows django_signposting to extract JSON-LD embedded in HTML `<script type="application/ld+json">` tags
+and add the corresponding signposting headers.
+It’s compatible with tools that provide JSON-LD, such as [django-json-ld](https://pypi.org/project/django-json-ld/).
+
+> Note: The middleware order is important! Place `SignpostingMiddleware` before `JsonLdSignpostingParserMiddleware` to ensure proper extraction and processing of JSON-LD content.
+
+### Manual signposting
+
+For cases where JSON-LD is not embedded, or you want to specify headers manually, you can use the `add_signposts` utility.
+
+1. **Add Middleware**: Add the `SignpostingMiddleware` to your Django project's `MIDDLEWARE` setting in `settings.py`:
 
 ```python
 MIDDLEWARE = [
@@ -34,10 +56,7 @@ MIDDLEWARE = [
 ]
 ```
 
-### 2. Add Signposts to your Views
+2. **Add Signposts to your Views:** Use the `add_signposts` utility function:
-
-You can add signposting headers in your Django views using the provided `add_signposts` utility function.
-Here's how you can use it:
 
 ```python
 from django.http import HttpResponse
@@ -58,7 +77,7 @@ def my_view(request):
     return response
 ```
 
-### 3. Signposts are formatted and added as Link headers by the middleware:
+## Signposts are formatted and added as Link headers by the middleware
 
 ```bash
 curl -I http://localhost:8000
@@ -69,9 +88,8 @@ link: <https://schema.org/Dataset> ; rel="type" ,
       <https://example.com/download.zip> ; rel="item" ; type="application/zip"
 ```
 
-### TODO
+## TODO
 
-- [ ] Automatically generate signposts from present JSON+LD.
 - [ ] Option to add signposts in HTML via <link> elements.
 
 

django_signposting/middleware.py

@@ -1,10 +1,17 @@
 from typing import Callable
 from django.http import HttpRequest, HttpResponse
-from signposting import Signpost
+from signposting import Signpost, LinkRel
+
+from bs4 import BeautifulSoup
+import re
+import json
+from django.utils.deprecation import MiddlewareMixin
+from django.conf import settings
+from rdflib import Graph
+from . import sparql
 
 
 class SignpostingMiddleware:
-
     def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]):
         self.get_response = get_response
 
@@ -17,7 +24,6 @@ class SignpostingMiddleware:
 
         if not hasattr(response, "_signposts"):
             return response
-
         self._add_signposts(response, response._signposts)
 
         return response
@@ -36,3 +42,88 @@ class SignpostingMiddleware:
                 link_snippets[-1] += f' ; type="{signpost.type}"'
 
         response["Link"] = " , ".join(link_snippets)
+
+
+class JsonLdSignpostingParserMiddleware(MiddlewareMixin):
+    def is_url(self, url: str) -> bool:
+        url_pattern = re.compile(
+            r"^(https?|ftp)://"  # protocol
+            r"(?:(?:[a-zA-Z0-9-_]+\.)?[a-zA-Z0-9-]+\.[a-zA-Z]{2,6})"  # domain
+            r"(?::\d{1,5})?"  # optional port
+            r"(?:/.*)?$"  # path
+        )
+        return bool(url_pattern.match(url))
+
+    def select_url(self, elements: tuple[str, ...]) -> str | None:
+        for elem in elements[::-1]:
+            if self.is_url(str(elem)):
+                return str(elem)
+        return None
+
+    def _jsonld_to_signposts(self, jsonld: dict) -> dict:
+        signposts = []
+        # TODO use jsonld context in query as prefix
+        g = Graph().parse(data=json.dumps(jsonld), format="json-ld")
+
+        rootElement = next(iter(sparql.root_element_query(g)), None)
+        if rootElement:
+            rootElement = rootElement[0]
+        else:
+            print("No root element found")
+            return {}
+
+        types = sparql.type_query(g, rootElement)
+        for type in types:
+            signposts.append(Signpost(LinkRel.type, str(type[0])))
+
+        authors = sparql.author_query(g, rootElement)
+        for author in authors:
+            author = self.select_url(author)
+            if author:
+                signposts.append(Signpost(LinkRel.author, author))
+
+        license = next(iter(sparql.license_query(g, rootElement)), [])
+        license = self.select_url(license)
+        if license:
+            signposts.append(Signpost(LinkRel.license, license))
+
+        citations = sparql.cite_query(g, rootElement)
+        for citation in citations:
+            citation = self.select_url(citation)
+            if citation:
+                signposts.append(Signpost(LinkRel.cite_as, citation))
+
+        sameas = sparql.sameas_query(g, rootElement)
+        for sa in sameas:
+            sa_media_type = sa[-1]
+            sa = self.select_url(sa[:-1])
+            if sa:
+                signposts.append(Signpost(LinkRel.describedby, sa, sa_media_type))
+
+        items = sparql.item_query(g, rootElement)
+        for item in items:
+            item_media_type = item[-1]
+            item = self.select_url(item[:-1])
+            if item:
+                signposts.append(Signpost(LinkRel.item, item, item_media_type))
+        return signposts
+
+    def process_response(
+        self, request: HttpRequest, response: HttpResponse
+    ) -> HttpResponse:
+        if not getattr(settings, "SIGNPOSTING_PARSE_JSONLD", True):
+            return response
+
+        if response.get("Content-Type", "").startswith("text/html"):
+            soup = BeautifulSoup(response.content, "html.parser")
+            for script in soup.find_all("script", type="application/ld+json"):
+                try:
+                    jsonld = json.loads(script.string)
+                    signposts = self._jsonld_to_signposts(jsonld)
+
+                    response._signposts = signposts
+                except json.JSONDecodeError as e:
+                    print(e)
+                    continue
+
+        return response

django_signposting/sparql.py

@@ -0,0 +1,136 @@
+from rdflib import Graph
+from rdflib.query import Result
+
+
+def root_element_query(g: Graph) -> Result:
+    return g.query("""
+PREFIX schema: <http://schema.org/>
+
+SELECT DISTINCT ?rootElement
+WHERE {
+  # has a type
+  ?rootElement a ?type ;
+  # has a license or an author or a creator
+  (schema:license | schema:author | schema:creator ) ?value .
+
+  # No incoming edges for ?rootElement
+  FILTER NOT EXISTS { ?s ?p ?rootElement }
+}
+LIMIT 1
+""")
+
+
+def type_query(g: Graph, rootElement: str) -> Graph:
+    return g.query(
+        """
+PREFIX schema: <?context>
+
+SELECT DISTINCT ?type
+WHERE {
+  ?rootElement a ?type .
+}
+""",
+        initBindings={"rootElement": rootElement},
+    )
+
+
+def author_query(g: Graph, rootElement: str):
+    return g.query(
+        """
+PREFIX schema: <http://schema.org/>
+
+SELECT ?author_id ?identifier ?url
+WHERE {
+    ?rootElement schema:author ?author .
+
+    BIND(?author AS ?author_id)  # Get the @id of the author
+    OPTIONAL { ?author schema:identifier ?identifier }
+    OPTIONAL { ?author schema:url ?url }
+}
+LIMIT 1
+
+""",
+        initBindings={"rootElement": rootElement},
+    )
+
+
+def license_query(g: Graph, rootElement: str):
+    return g.query(
+        """
+PREFIX schema: <http://schema.org/>
+
+SELECT ?element_id ?identifier ?url
+WHERE {
+    ?rootElement schema:license ?element .
+
+    BIND(?element AS ?element_id)  # Get the @id of the element
+    OPTIONAL { ?element schema:identifier ?identifier }
+    OPTIONAL { ?element schema:url ?url }
+}
+LIMIT 1
+""",
+        initBindings={"rootElement": rootElement},
+    )
+
+
+def cite_query(g: Graph, rootElement: str):
+    return g.query(
+        """
+PREFIX schema: <http://schema.org/>
+
+SELECT ?element_id ?identifier ?url
+WHERE {
+    ?rootElement schema:url ?element .
+
+    BIND(?element AS ?element_id)  # Get the @id of the element
+    OPTIONAL { ?element schema:identifier ?identifier }
+    OPTIONAL { ?element schema:url ?url }
+}
+LIMIT 1
+
+""",
+        initBindings={"rootElement": rootElement},
+    )
+
+
+def sameas_query(g: Graph, rootElement: str):
+    return g.query(
+        """
+PREFIX schema: <http://schema.org/>
+
+SELECT ?element_id ?contentUrl ?identifier ?url ?encoding
+WHERE {
+    ?rootElement schema:sameAs ?element .
+
+    BIND(?element AS ?element_id)  # Get the @id of the element
+    OPTIONAL { ?element schema:contentUrl ?contentUrl }
+    OPTIONAL { ?element schema:identifier ?identifier }
+    OPTIONAL { ?element schema:url ?url }
+    OPTIONAL { ?element schema:encodingFormat ?encoding }
+}
+""",
+        initBindings={"rootElement": rootElement},
+    )
+
+
+def item_query(g: Graph, rootElement: str):
+    return g.query(
+        """
+PREFIX schema: <http://schema.org/>
+
+SELECT ?element_id ?identifier ?url ?contentUrl ?encoding
+WHERE {
+    ?rootElement schema:hasPart ?element .
+    #VALUES ?element_type { schema:MediaObject schema:Dataset }
+    #?element a ?element_type .
+
+    BIND(?element AS ?element_id)  # Get the @id of the element
+    OPTIONAL { ?element schema:contentUrl ?contentUrl }
+    OPTIONAL { ?element schema:identifier ?identifier }
+    OPTIONAL { ?element schema:url ?url }
+    OPTIONAL { ?element schema:encodingFormat ?encoding }
+}
+
+""",
+        initBindings={"rootElement": rootElement},
+    )

example/example/settings.py

@@ -37,6 +37,7 @@ INSTALLED_APPS = [
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django_json_ld',
 ]
 
 MIDDLEWARE = [
@@ -48,6 +49,7 @@ MIDDLEWARE = [
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django_signposting.middleware.SignpostingMiddleware',
+    'django_signposting.middleware.JsonLdSignpostingParserMiddleware',
 ]
 
 ROOT_URLCONF = 'example.urls'
@@ -55,7 +57,9 @@ ROOT_URLCONF = 'example.urls'
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [],
+        'DIRS': [
+            BASE_DIR / 'example/templates',
+        ],
         'APP_DIRS': True,
         'OPTIONS': {
             'context_processors': [

example/example/templates/jsonld.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+<head>
+{% load render_json_ld from json_ld %}
+{% render_json_ld sd %}
+</head>
+<body>
+    <p>Hello, world!</p>
+</body>
+</html>
+

example/example/urls.py

@@ -19,5 +19,6 @@ from django.urls import path
 from . import views
 
 urlpatterns = [
-    path("", views.my_view),
+    path("", views.SimpleView.as_view()),
+    path("jsonld", views.JsonLdView.as_view()),
 ]

example/example/views.py

@@ -1,10 +1,16 @@
 from django.http import HttpResponse
+from django.views import View
+from django.shortcuts import render
+
+from django_json_ld.views import JsonLdContextMixin
 from django_signposting.utils import add_signposts
 
 from signposting import Signpost, LinkRel
 
 
-def my_view(request):
+class SimpleView(View):
+
+    def get(self, request):
         response = HttpResponse("Hello, world!")
 
         # Add signpostings as string
@@ -15,3 +21,50 @@ def my_view(request):
         )
 
         return response
+
+
+class JsonLdView(JsonLdContextMixin, View):
+
+    sd = {
+        "@context": "https://schema.org",
+        "@type": ["WebSite", "Dataset"],
+        "name": "My Dataset",
+        "description": "A dataset of things.",
+        "url": "https://example.com",
+        "sameAs": [
+            {
+                "@type": "MediaObject",
+                "contentUrl": "https://example.com/download.zip",
+                "encodingFormat": "application/zip"
+            },
+            {
+                "@type": "MediaObject",
+                "contentUrl": "https://example.com/metadata.json",
+            }
+        ],
+        "author": {
+            "@type": "Person",
+            "name": "Daniel Bauer",
+            "url": "https://orcid.org/0000-0001-9447-460X",
+        },
+        "license": {
+            "@type": "CreativeWork",
+            "name": "CC BY 4.0",
+            "url": "https://creativecommons.org/licenses/by/4.0/"
+        },
+        "hasPart": [
+            {
+                "@type": "ImageObject",
+                "url": "http://example.com/image.png",
+                "encodingFormat": "image/png"
+            },
+            {
+                "@type": "ImageObject",
+                "url": "http://example.com/image2.png",
+                "encodingFormat": "image/png"
+            }
+        ]
+    }
+
+    def get(self, request):
+        return render(request, "jsonld.html", context={"sd": self.sd})

pyproject.toml

@@ -12,8 +12,10 @@ version = "0.10.0"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    "Django>=3.0",
+    "beautifulsoup4>=4.12.3",
+    "django>=3.0",
     "signposting>=0.9.9",
+    "rdflib>=7.1.1",
 ]
 license = {file= "LICENSE"}
 

tests/test_jsonld_signposting_middleware.py

@@ -0,0 +1,220 @@
+import json
+from django.http import HttpRequest, HttpResponse
+
+from django_signposting.middleware import JsonLdSignpostingParserMiddleware
+from signposting import Signpost, LinkRel
+
+
+def jsonld_test_runner(jsonld, expected_signposts):
+    response = HttpResponse(f"""
+    <html><head>
+    <script type="application/ld+json">{json.dumps(jsonld)}</script>
+    </head>
+    <body></body>
+    </html>
+    """)
+    response.status_code = 200
+
+    middleware = JsonLdSignpostingParserMiddleware(lambda request: response)
+    response = middleware(HttpRequest())
+
+    for expected_signpost in expected_signposts:
+        assert expected_signpost in response._signposts
+
+    assert len(expected_signposts) == len(response._signposts)
+
+
+def test_jsonld_signposting_basic():
+    jsonld_test_runner(
+        {
+            "@context": "http://schema.org",
+            "@type": "WebPage",
+            "author": {"url": "http://example.com/author"},
+            "url": "http://example.com/url",
+            "license": {"identifier": "http://example.com/license"},
+            "hasPart": [
+                {
+                    "url": "http://example.com/image1.jpg",
+                },
+                {
+                    "url": "http://example.com/image2.jpg",
+                },
+            ],
+            "sameAs": [
+                {
+                    "url": "http://example.com/metadata.json",
+                },
+                {
+                    "url": "http://example.com/metadata.xml",
+                },
+            ],
+        },
+        [
+            Signpost(LinkRel.type, "http://schema.org/WebPage"),
+            Signpost(LinkRel.author, "http://example.com/author"),
+            Signpost(LinkRel.cite_as, "http://example.com/url"),
+            Signpost(LinkRel.license, "http://example.com/license"),
+            Signpost(LinkRel.item, "http://example.com/image1.jpg"),
+            Signpost(LinkRel.item, "http://example.com/image2.jpg"),
+            Signpost(LinkRel.describedby, "http://example.com/metadata.json"),
+            Signpost(LinkRel.describedby, "http://example.com/metadata.xml"),
+        ],
+    )
+
+
+def test_jsonld_empty_signposting():
+    jsonld_test_runner({}, [])
+
+
+def test_jsonld_signposting_media_types():
+    jsonld_test_runner(
+        {
+            "@context": "http://schema.org",
+            "@type": "WebPage",
+            "author": {"url": "http://example.com/author"},
+            "url": "http://example.com/url",
+            "license": {"identifier": "http://example.com/license"},
+            "hasPart": [
+                {
+                    "url": "http://example.com/image1.jpg",
+                    "encodingFormat": "image/jpeg",
+                },
+                {"url": "http://example.com/image2.jpg", "encodingFormat": "image/png"},
+            ],
+            "sameAs": [
+                {
+                    "url": "http://example.com/metadata.json",
+                    "encodingFormat": "application/json",
+                },
+                {
+                    "url": "http://example.com/metadata.xml",
+                    "encodingFormat": "application/xml",
+                },
+            ],
+        },
+        [
+            Signpost(LinkRel.type, "http://schema.org/WebPage"),
+            Signpost(LinkRel.author, "http://example.com/author"),
+            Signpost(LinkRel.cite_as, "http://example.com/url"),
+            Signpost(LinkRel.license, "http://example.com/license"),
+            Signpost(LinkRel.item, "http://example.com/image1.jpg", "image/jpeg"),
+            Signpost(LinkRel.item, "http://example.com/image2.jpg", "image/png"),
+            Signpost(
+                LinkRel.describedby,
+                "http://example.com/metadata.json",
+                "application/json",
+            ),
+            Signpost(
+                LinkRel.describedby,
+                "http://example.com/metadata.xml",
+                "application/xml",
+            ),
+        ],
+    )
+
+
+def test_jsonld_signposting_property_precedence():
+    jsonld_test_runner(
+        {
+            "@context": "http://schema.org",
+            "@type": "WebPage",
+            "author": {
+                "identifier": "http://example.com/author-ident",
+                "url": "http://example.com/author-url",
+            },
+            "hasPart": [
+                {
+                    "identifier": "http://example.com/image1-identifier.jpg",
+                    "url": "http://example.com/image1-url.jpg",
+                    "contentUrl": "http://example.com/image1-content.jpg",
+                },
+                {
+                    "identifier": "http://example.com/image2-identifier.jpg",
+                    "url": "http://example.com/image2-url.jpg",
+                },
+            ],
+        },
+        [
+            Signpost(LinkRel.type, "http://schema.org/WebPage"),
+            Signpost(LinkRel.author, "http://example.com/author-url"),
+            Signpost(LinkRel.item, "http://example.com/image1-content.jpg"),
+            Signpost(LinkRel.item, "http://example.com/image2-url.jpg"),
+        ],
+    )
+
+
+def test_jsonld_signposting_ids():
+    jsonld_test_runner(
+        {
+            "@context": "http://schema.org",
+            "@graph": [
+                {
+                    "@type": "WebPage",
+                    "author": {"@id": "http://example.com/author"},
+                    "url": "http://example.com/url",
+                    "license": {"@id": "http://example.com/license"},
+                    "hasPart": [
+                        {
+                            "@id": "http://example.com/image1.jpg",
+                        },
+                    ],
+                    "sameAs": [
+                        {
+                            "@id": "http://example.com/metadata.json",
+                        },
+                    ],
+                },
+                {"@id": "http://example.com/author", "@type": "Person"},
+                {"@id": "http://example.com/license"},
+                {
+                    "@id": "http://example.com/image1.jpg",
+                    "encodingFormat": "image/jpeg",
+                },
+                {"@id": "http://example.com/metadata.json"},
+            ],
+        },
+        [
+            Signpost(LinkRel.type, "http://schema.org/WebPage"),
+            Signpost(LinkRel.author, "http://example.com/author"),
+            Signpost(LinkRel.cite_as, "http://example.com/url"),
+            Signpost(LinkRel.license, "http://example.com/license"),
+            Signpost(LinkRel.item, "http://example.com/image1.jpg", "image/jpeg"),
+            Signpost(LinkRel.describedby, "http://example.com/metadata.json"),
+        ],
+    )
+
+
+def test_jsonld_signposting_ids_with_url():
+    jsonld_test_runner(
+        {
+            "@context": "http://schema.org",
+            "@graph": [
+                {
+                    "@type": "WebPage",
+                    "author": {"@id": "http://example.com/author"},
+                },
+                {
+                    "@id": "http://example.com/author",
+                    "url": "http://example.com/realAuthorUrl",
+                },
+            ],
+        },
+        [
+            Signpost(LinkRel.type, "http://schema.org/WebPage"),
+            Signpost(LinkRel.author, "http://example.com/realAuthorUrl"),
+        ],
+    )
+
+
+def test_jsonld_signposting_multiple_types():
+    jsonld_test_runner(
+        {
+            "@context": "http://schema.org",
+            "@type": "Dataset",
+            "author": "http://example.com/author",
+        },
+        [
+            Signpost(LinkRel.type, "http://schema.org/Dataset"),
+            Signpost(LinkRel.author, "http://example.com/author"),
+        ],
+    )