bauer/django-signposting
1
0
Fork 0
mirror of https://github.com/dnlbauer/django-signposting.git synced 2026-09-10 14:05:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extract signposts from jsonld (#1)

Browse Source 
* use rdflib to extract signposts from jsonld

* use rdflib to extract signposts from jsonld

* linting

* unit tests for jsonld middleware

* update readme

* update flake pipeline

---------

Co-authored-by: Daniel Bauer <daniel.bauer@senckenberg.de>
This commit is contained in:
Daniel Bauer
2024-11-11 13:02:55 +01:00
committed by GitHub
parent bbf0527733
commit 0cd321112f
11 changed files with 564 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/python-package.yml vendored
View File

@@ -32,7 +32,7 @@ jobs:
python -m pip install -e '.[dev]'
- name: Lint with flake8
run: |
flake8 . --count --max-complexity=10 --max-line-length=127 --statistics
flake8 . --count --max-line-length=127 --statistics
- name: Test with pytest
run: |
pytest

38
README.md
View File

@@ -1,6 +1,6 @@
[![Python package](https://github.com/dnlbauer/django-signposting/actions/workflows/python-package.yml/badge.svg)](https://github.com/dnlbauer/django-signposting/actions/workflows/python-package.yml)
# FAIR signposting for Django
# FAIR signposting middleware for Django
`django_signposting` is a Django middleware library that facilitates the addition of
FAIR signposting headers to HTTP responses.
@@ -11,6 +11,7 @@ Based on the [Signposting](https://github.com/stain/signposting) library.
## Features
- Automatically adds signposting headers to HTTP responses.
- Signposts can be added manually or automatically be parsed from JSON-LD/schema.org
- Supports multiple relation types with optional media type specification.
- Easily integrable with existing Django applications.
@@ -22,9 +23,30 @@ pip install django_signposting
## Usage
### 1. Add Middleware
### Automatic parsing of JSON-LD
Add the middleware to your Django project's `MIDDLEWARE` setting in `settings.py`:
To enable automatic parsing of JSON-LD, add the following middleware classes to your Django project's MIDDLEWARE setting in settings.py:
```python
MIDDLEWARE = [
...,
'django_signposting.middleware.SignpostingMiddleware',
'django_signposting.middleware.JsonLdSignpostingParserMiddleware',
...,
]
```
This setup allows django_signposting to extract JSON-LD embedded in HTML `<script type="application/ld+json">` tags
and add the corresponding signposting headers.
It’s compatible with tools that provide JSON-LD, such as [django-json-ld](https://pypi.org/project/django-json-ld/).
> Note: The middleware order is important! Place `SignpostingMiddleware` before `JsonLdSignpostingParserMiddleware` to ensure proper extraction and processing of JSON-LD content.
### Manual signposting
For cases where JSON-LD is not embedded, or you want to specify headers manually, you can use the `add_signposts` utility.
1. **Add Middleware**: Add the `SignpostingMiddleware` to your Django project's `MIDDLEWARE` setting in `settings.py`:
```python
MIDDLEWARE = [
@@ -34,10 +56,7 @@ MIDDLEWARE = [
]
```
### 2. Add Signposts to your Views
You can add signposting headers in your Django views using the provided `add_signposts` utility function.
Here's how you can use it:
2. **Add Signposts to your Views:** Use the `add_signposts` utility function:
```python
from django.http import HttpResponse
@@ -58,7 +77,7 @@ def my_view(request):
return response
```
### 3. Signposts are formatted and added as Link headers by the middleware:
## Signposts are formatted and added as Link headers by the middleware
```bash
curl -I http://localhost:8000
@@ -69,9 +88,8 @@ link: <https://schema.org/Dataset> ; rel="type" ,
<https://example.com/download.zip> ; rel="item" ; type="application/zip"
```
### TODO
## TODO
- [ ] Automatically generate signposts from present JSON+LD.
- [ ] Option to add signposts in HTML via <link> elements.

99
django_signposting/middleware.py
View File

@@ -1,10 +1,17 @@
from typing import Callable
from django.http import HttpRequest, HttpResponse
from signposting import Signpost
from signposting import Signpost, LinkRel
from bs4 import BeautifulSoup
import re
import json
from django.utils.deprecation import MiddlewareMixin
from django.conf import settings
from rdflib import Graph
from . import sparql
class SignpostingMiddleware:
def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]):
self.get_response = get_response
@@ -17,13 +24,12 @@ class SignpostingMiddleware:
if not hasattr(response, "_signposts"):
return response
self._add_signposts(response, response._signposts)
return response
def _add_signposts(self, response: HttpResponse, signposts: list[Signpost]):
""" Adds signposting headers to the respones.
"""Adds signposting headers to the respones.
params:
response - the response object
signposts - a list of Signposts
@@ -36,3 +42,88 @@ class SignpostingMiddleware:
link_snippets[-1] += f' ; type="{signpost.type}"'
response["Link"] = " , ".join(link_snippets)
class JsonLdSignpostingParserMiddleware(MiddlewareMixin):
def is_url(self, url: str) -> bool:
url_pattern = re.compile(
r"^(https?|ftp)://" # protocol
r"(?:(?:[a-zA-Z0-9-_]+\.)?[a-zA-Z0-9-]+\.[a-zA-Z]{2,6})" # domain
r"(?::\d{1,5})?" # optional port
r"(?:/.*)?$" # path
)
return bool(url_pattern.match(url))
def select_url(self, elements: tuple[str, ...]) -> str | None:
for elem in elements[::-1]:
if self.is_url(str(elem)):
return str(elem)
return None
def _jsonld_to_signposts(self, jsonld: dict) -> dict:
signposts = []
# TODO use jsonld context in query as prefix
g = Graph().parse(data=json.dumps(jsonld), format="json-ld")
rootElement = next(iter(sparql.root_element_query(g)), None)
if rootElement:
rootElement = rootElement[0]
else:
print("No root element found")
return {}
types = sparql.type_query(g, rootElement)
for type in types:
signposts.append(Signpost(LinkRel.type, str(type[0])))
authors = sparql.author_query(g, rootElement)
for author in authors:
author = self.select_url(author)
if author:
signposts.append(Signpost(LinkRel.author, author))
license = next(iter(sparql.license_query(g, rootElement)), [])
license = self.select_url(license)
if license:
signposts.append(Signpost(LinkRel.license, license))
citations = sparql.cite_query(g, rootElement)
for citation in citations:
citation = self.select_url(citation)
if citation:
signposts.append(Signpost(LinkRel.cite_as, citation))
sameas = sparql.sameas_query(g, rootElement)
for sa in sameas:
sa_media_type = sa[-1]
sa = self.select_url(sa[:-1])
if sa:
signposts.append(Signpost(LinkRel.describedby, sa, sa_media_type))
items = sparql.item_query(g, rootElement)
for item in items:
item_media_type = item[-1]
item = self.select_url(item[:-1])
if item:
signposts.append(Signpost(LinkRel.item, item, item_media_type))
return signposts
def process_response(
self, request: HttpRequest, response: HttpResponse
) -> HttpResponse:
if not getattr(settings, "SIGNPOSTING_PARSE_JSONLD", True):
return response
if response.get("Content-Type", "").startswith("text/html"):
soup = BeautifulSoup(response.content, "html.parser")
for script in soup.find_all("script", type="application/ld+json"):
try:
jsonld = json.loads(script.string)
signposts = self._jsonld_to_signposts(jsonld)
response._signposts = signposts
except json.JSONDecodeError as e:
print(e)
continue
return response

136
django_signposting/sparql.py Normal file
View File

@@ -0,0 +1,136 @@
from rdflib import Graph
from rdflib.query import Result
def root_element_query(g: Graph) -> Result:
return g.query("""
PREFIX schema: <http://schema.org/>
SELECT DISTINCT ?rootElement
WHERE {
# has a type
?rootElement a ?type ;
# has a license or an author or a creator
(schema:license | schema:author | schema:creator ) ?value .
# No incoming edges for ?rootElement
FILTER NOT EXISTS { ?s ?p ?rootElement }
}
LIMIT 1
""")
def type_query(g: Graph, rootElement: str) -> Graph:
return g.query(
"""
PREFIX schema: <?context>
SELECT DISTINCT ?type
WHERE {
?rootElement a ?type .
}
""",
initBindings={"rootElement": rootElement},
)
def author_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?author_id ?identifier ?url
WHERE {
?rootElement schema:author ?author .
BIND(?author AS ?author_id) # Get the @id of the author
OPTIONAL { ?author schema:identifier ?identifier }
OPTIONAL { ?author schema:url ?url }
}
LIMIT 1
""",
initBindings={"rootElement": rootElement},
)
def license_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?element_id ?identifier ?url
WHERE {
?rootElement schema:license ?element .
BIND(?element AS ?element_id) # Get the @id of the element
OPTIONAL { ?element schema:identifier ?identifier }
OPTIONAL { ?element schema:url ?url }
}
LIMIT 1
""",
initBindings={"rootElement": rootElement},
)
def cite_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?element_id ?identifier ?url
WHERE {
?rootElement schema:url ?element .
BIND(?element AS ?element_id) # Get the @id of the element
OPTIONAL { ?element schema:identifier ?identifier }
OPTIONAL { ?element schema:url ?url }
}
LIMIT 1
""",
initBindings={"rootElement": rootElement},
)
def sameas_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?element_id ?contentUrl ?identifier ?url ?encoding
WHERE {
?rootElement schema:sameAs ?element .
BIND(?element AS ?element_id) # Get the @id of the element
OPTIONAL { ?element schema:contentUrl ?contentUrl }
OPTIONAL { ?element schema:identifier ?identifier }
OPTIONAL { ?element schema:url ?url }
OPTIONAL { ?element schema:encodingFormat ?encoding }
}
""",
initBindings={"rootElement": rootElement},
)
def item_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?element_id ?identifier ?url ?contentUrl ?encoding
WHERE {
?rootElement schema:hasPart ?element .
#VALUES ?element_type { schema:MediaObject schema:Dataset }
#?element a ?element_type .
BIND(?element AS ?element_id) # Get the @id of the element
OPTIONAL { ?element schema:contentUrl ?contentUrl }
OPTIONAL { ?element schema:identifier ?identifier }
OPTIONAL { ?element schema:url ?url }
OPTIONAL { ?element schema:encodingFormat ?encoding }
}
""",
initBindings={"rootElement": rootElement},
)

6
example/example/settings.py
View File

@@ -37,6 +37,7 @@ INSTALLED_APPS = [
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django_json_ld',
]
MIDDLEWARE = [
@@ -48,6 +49,7 @@ MIDDLEWARE = [
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django_signposting.middleware.SignpostingMiddleware',
'django_signposting.middleware.JsonLdSignpostingParserMiddleware',
]
ROOT_URLCONF = 'example.urls'
@@ -55,7 +57,9 @@ ROOT_URLCONF = 'example.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'DIRS': [
BASE_DIR / 'example/templates',
],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [

11
example/example/templates/jsonld.html Normal file
View File

@@ -0,0 +1,11 @@
<!DOCTYPE html>
<html>
<head>
{% load render_json_ld from json_ld %}
{% render_json_ld sd %}
</head>
<body>
<p>Hello, world!</p>
</body>
</html>

3
example/example/urls.py
View File

@@ -19,5 +19,6 @@ from django.urls import path
from . import views
urlpatterns = [
path("", views.my_view),
path("", views.SimpleView.as_view()),
path("jsonld", views.JsonLdView.as_view()),
]

71
example/example/views.py
View File

@@ -1,17 +1,70 @@
from django.http import HttpResponse
from django.views import View
from django.shortcuts import render
from django_json_ld.views import JsonLdContextMixin
from django_signposting.utils import add_signposts
from signposting import Signpost, LinkRel
def my_view(request):
response = HttpResponse("Hello, world!")
class SimpleView(View):
# Add signpostings as string
add_signposts(
response,
Signpost(LinkRel.type, "http://schema.org/Dataset"),
Signpost(LinkRel.author, "https://orcid.org/0000-0001-9447-460X")
)
def get(self, request):
response = HttpResponse("Hello, world!")
return response
# Add signpostings as string
add_signposts(
response,
Signpost(LinkRel.type, "http://schema.org/Dataset"),
Signpost(LinkRel.author, "https://orcid.org/0000-0001-9447-460X")
)
return response
class JsonLdView(JsonLdContextMixin, View):
sd = {
"@context": "https://schema.org",
"@type": ["WebSite", "Dataset"],
"name": "My Dataset",
"description": "A dataset of things.",
"url": "https://example.com",
"sameAs": [
{
"@type": "MediaObject",
"contentUrl": "https://example.com/download.zip",
"encodingFormat": "application/zip"
},
{
"@type": "MediaObject",
"contentUrl": "https://example.com/metadata.json",
}
],
"author": {
"@type": "Person",
"name": "Daniel Bauer",
"url": "https://orcid.org/0000-0001-9447-460X",
},
"license": {
"@type": "CreativeWork",
"name": "CC BY 4.0",
"url": "https://creativecommons.org/licenses/by/4.0/"
},
"hasPart": [
{
"@type": "ImageObject",
"url": "http://example.com/image.png",
"encodingFormat": "image/png"
},
{
"@type": "ImageObject",
"url": "http://example.com/image2.png",
"encodingFormat": "image/png"
}
]
}
def get(self, request):
return render(request, "jsonld.html", context={"sd": self.sd})

6
pyproject.toml
View File

@@ -12,8 +12,10 @@ version = "0.10.0"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"Django>=3.0",
"beautifulsoup4>=4.12.3",
"django>=3.0",
"signposting>=0.9.9",
"rdflib>=7.1.1",
]
license = {file= "LICENSE"}
@@ -28,4 +30,4 @@ dev = [
[pytest]
minversion = 6.0
addopts = "-ra -q"
testpaths = ["tests"]
testpaths = ["tests"]

220
tests/test_jsonld_signposting_middleware.py Normal file
View File

@@ -0,0 +1,220 @@
import json
from django.http import HttpRequest, HttpResponse
from django_signposting.middleware import JsonLdSignpostingParserMiddleware
from signposting import Signpost, LinkRel
def jsonld_test_runner(jsonld, expected_signposts):
response = HttpResponse(f"""
<html><head>
<script type="application/ld+json">{json.dumps(jsonld)}</script>
</head>
<body></body>
</html>
""")
response.status_code = 200
middleware = JsonLdSignpostingParserMiddleware(lambda request: response)
response = middleware(HttpRequest())
for expected_signpost in expected_signposts:
assert expected_signpost in response._signposts
assert len(expected_signposts) == len(response._signposts)
def test_jsonld_signposting_basic():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@type": "WebPage",
"author": {"url": "http://example.com/author"},
"url": "http://example.com/url",
"license": {"identifier": "http://example.com/license"},
"hasPart": [
{
"url": "http://example.com/image1.jpg",
},
{
"url": "http://example.com/image2.jpg",
},
],
"sameAs": [
{
"url": "http://example.com/metadata.json",
},
{
"url": "http://example.com/metadata.xml",
},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/author"),
Signpost(LinkRel.cite_as, "http://example.com/url"),
Signpost(LinkRel.license, "http://example.com/license"),
Signpost(LinkRel.item, "http://example.com/image1.jpg"),
Signpost(LinkRel.item, "http://example.com/image2.jpg"),
Signpost(LinkRel.describedby, "http://example.com/metadata.json"),
Signpost(LinkRel.describedby, "http://example.com/metadata.xml"),
],
)
def test_jsonld_empty_signposting():
jsonld_test_runner({}, [])
def test_jsonld_signposting_media_types():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@type": "WebPage",
"author": {"url": "http://example.com/author"},
"url": "http://example.com/url",
"license": {"identifier": "http://example.com/license"},
"hasPart": [
{
"url": "http://example.com/image1.jpg",
"encodingFormat": "image/jpeg",
},
{"url": "http://example.com/image2.jpg", "encodingFormat": "image/png"},
],
"sameAs": [
{
"url": "http://example.com/metadata.json",
"encodingFormat": "application/json",
},
{
"url": "http://example.com/metadata.xml",
"encodingFormat": "application/xml",
},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/author"),
Signpost(LinkRel.cite_as, "http://example.com/url"),
Signpost(LinkRel.license, "http://example.com/license"),
Signpost(LinkRel.item, "http://example.com/image1.jpg", "image/jpeg"),
Signpost(LinkRel.item, "http://example.com/image2.jpg", "image/png"),
Signpost(
LinkRel.describedby,
"http://example.com/metadata.json",
"application/json",
),
Signpost(
LinkRel.describedby,
"http://example.com/metadata.xml",
"application/xml",
),
],
)
def test_jsonld_signposting_property_precedence():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@type": "WebPage",
"author": {
"identifier": "http://example.com/author-ident",
"url": "http://example.com/author-url",
},
"hasPart": [
{
"identifier": "http://example.com/image1-identifier.jpg",
"url": "http://example.com/image1-url.jpg",
"contentUrl": "http://example.com/image1-content.jpg",
},
{
"identifier": "http://example.com/image2-identifier.jpg",
"url": "http://example.com/image2-url.jpg",
},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/author-url"),
Signpost(LinkRel.item, "http://example.com/image1-content.jpg"),
Signpost(LinkRel.item, "http://example.com/image2-url.jpg"),
],
)
def test_jsonld_signposting_ids():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@graph": [
{
"@type": "WebPage",
"author": {"@id": "http://example.com/author"},
"url": "http://example.com/url",
"license": {"@id": "http://example.com/license"},
"hasPart": [
{
"@id": "http://example.com/image1.jpg",
},
],
"sameAs": [
{
"@id": "http://example.com/metadata.json",
},
],
},
{"@id": "http://example.com/author", "@type": "Person"},
{"@id": "http://example.com/license"},
{
"@id": "http://example.com/image1.jpg",
"encodingFormat": "image/jpeg",
},
{"@id": "http://example.com/metadata.json"},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/author"),
Signpost(LinkRel.cite_as, "http://example.com/url"),
Signpost(LinkRel.license, "http://example.com/license"),
Signpost(LinkRel.item, "http://example.com/image1.jpg", "image/jpeg"),
Signpost(LinkRel.describedby, "http://example.com/metadata.json"),
],
)
def test_jsonld_signposting_ids_with_url():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@graph": [
{
"@type": "WebPage",
"author": {"@id": "http://example.com/author"},
},
{
"@id": "http://example.com/author",
"url": "http://example.com/realAuthorUrl",
},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/realAuthorUrl"),
],
)
def test_jsonld_signposting_multiple_types():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@type": "Dataset",
"author": "http://example.com/author",
},
[
Signpost(LinkRel.type, "http://schema.org/Dataset"),
Signpost(LinkRel.author, "http://example.com/author"),
],
)

0
tests/test_middleware.py → tests/test_signposting_middleware.py
View File