8 Commits

Author SHA1 Message Date
Daniel Bauer
7b93eae884 Update pyproject.toml 2024-11-18 17:23:53 +01:00
Daniel Bauer
0cd321112f Extract signposts from jsonld (#1)
* use rdflib to extract signposts from jsonld

* use rdflib to extract signposts from jsonld

* linting

* unit tests for jsonld middleware

* update readme

* update flake pipeline

---------

Co-authored-by: Daniel Bauer <daniel.bauer@senckenberg.de>
2024-11-11 13:02:55 +01:00
Daniel Bauer
bbf0527733 more agressive lint checking in pipeline 2024-11-11 09:45:26 +01:00
Daniel Bauer
8b09f4feb5 linting 2024-11-11 09:40:52 +01:00
Daniel Bauer
01affafd7a requirements file for example 2024-11-08 17:47:09 +01:00
Daniel Bauer
76e791914d update example to use signposting.Signpost 2024-11-08 17:44:48 +01:00
Daniel Bauer
cbd621435c Update README.md 2024-11-01 13:39:08 +01:00
Daniel Bauer
8c548d2392 add todos to readme 2024-11-01 13:17:56 +01:00
17 changed files with 601 additions and 38 deletions

3
.flake8 Normal file
View File

@@ -0,0 +1,3 @@
[flake8]
max-line-length = 99
exclude = .git, __pycache__, .pytest_cache, .venv, venv, build

View File

@@ -32,10 +32,7 @@ jobs:
python -m pip install -e '.[dev]'
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
flake8 . --count --max-line-length=127 --statistics
- name: Test with pytest
run: |
pytest

3
.pre-commit-config.yaml Normal file
View File

@@ -0,0 +1,3 @@
- repo: https://github.com/pycqa/flake8
hooks:
- id: flake8

View File

@@ -1,6 +1,6 @@
[![Python package](https://github.com/dnlbauer/django-signposting/actions/workflows/python-package.yml/badge.svg)](https://github.com/dnlbauer/django-signposting/actions/workflows/python-package.yml)
# FAIR signposting for Django
# FAIR signposting middleware for Django
`django_signposting` is a Django middleware library that facilitates the addition of
FAIR signposting headers to HTTP responses.
@@ -11,6 +11,7 @@ Based on the [Signposting](https://github.com/stain/signposting) library.
## Features
- Automatically adds signposting headers to HTTP responses.
- Signposts can be added manually or automatically be parsed from JSON-LD/schema.org
- Supports multiple relation types with optional media type specification.
- Easily integrable with existing Django applications.
@@ -22,9 +23,30 @@ pip install django_signposting
## Usage
### 1. Add Middleware
### Automatic parsing of JSON-LD
Add the middleware to your Django project's `MIDDLEWARE` setting in `settings.py`:
To enable automatic parsing of JSON-LD, add the following middleware classes to your Django project's MIDDLEWARE setting in settings.py:
```python
MIDDLEWARE = [
...,
'django_signposting.middleware.SignpostingMiddleware',
'django_signposting.middleware.JsonLdSignpostingParserMiddleware',
...,
]
```
This setup allows django_signposting to extract JSON-LD embedded in HTML `<script type="application/ld+json">` tags
and add the corresponding signposting headers.
Its compatible with tools that provide JSON-LD, such as [django-json-ld](https://pypi.org/project/django-json-ld/).
> Note: The middleware order is important! Place `SignpostingMiddleware` before `JsonLdSignpostingParserMiddleware` to ensure proper extraction and processing of JSON-LD content.
### Manual signposting
For cases where JSON-LD is not embedded, or you want to specify headers manually, you can use the `add_signposts` utility.
1. **Add Middleware**: Add the `SignpostingMiddleware` to your Django project's `MIDDLEWARE` setting in `settings.py`:
```python
MIDDLEWARE = [
@@ -34,10 +56,7 @@ MIDDLEWARE = [
]
```
### 2. Add Signposts to your Views
You can add signposting headers in your Django views using the provided `add_signposts` utility function.
Here's how you can use it:
2. **Add Signposts to your Views:** Use the `add_signposts` utility function:
```python
from django.http import HttpResponse
@@ -58,10 +77,10 @@ def my_view(request):
return response
```
### 3. Signposts are formatted and added as Link headers by the middleware:
## Signposts are formatted and added as Link headers by the middleware
```bash
curl -I https://example.com
curl -I http://localhost:8000
HTTP/2 200
...
link: <https://schema.org/Dataset> ; rel="type" ,
@@ -69,6 +88,11 @@ link: <https://schema.org/Dataset> ; rel="type" ,
<https://example.com/download.zip> ; rel="item" ; type="application/zip"
```
## TODO
- [ ] Option to add signposts in HTML via <link> elements.
## License
Licensed under the MIT License.

View File

@@ -1,10 +1,17 @@
from typing import Callable
from django.http import HttpRequest, HttpResponse
from signposting import Signpost
from signposting import Signpost, LinkRel
from bs4 import BeautifulSoup
import re
import json
from django.utils.deprecation import MiddlewareMixin
from django.conf import settings
from rdflib import Graph
from . import sparql
class SignpostingMiddleware:
def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]):
self.get_response = get_response
@@ -17,13 +24,12 @@ class SignpostingMiddleware:
if not hasattr(response, "_signposts"):
return response
self._add_signposts(response, response._signposts)
return response
def _add_signposts(self, response: HttpResponse, signposts: list[Signpost]):
""" Adds signposting headers to the respones.
"""Adds signposting headers to the respones.
params:
response - the response object
signposts - a list of Signposts
@@ -37,3 +43,87 @@ class SignpostingMiddleware:
response["Link"] = " , ".join(link_snippets)
class JsonLdSignpostingParserMiddleware(MiddlewareMixin):
def is_url(self, url: str) -> bool:
url_pattern = re.compile(
r"^(https?|ftp)://" # protocol
r"(?:(?:[a-zA-Z0-9-_]+\.)?[a-zA-Z0-9-]+\.[a-zA-Z]{2,6})" # domain
r"(?::\d{1,5})?" # optional port
r"(?:/.*)?$" # path
)
return bool(url_pattern.match(url))
def select_url(self, elements: tuple[str, ...]) -> str | None:
for elem in elements[::-1]:
if self.is_url(str(elem)):
return str(elem)
return None
def _jsonld_to_signposts(self, jsonld: dict) -> dict:
signposts = []
# TODO use jsonld context in query as prefix
g = Graph().parse(data=json.dumps(jsonld), format="json-ld")
rootElement = next(iter(sparql.root_element_query(g)), None)
if rootElement:
rootElement = rootElement[0]
else:
print("No root element found")
return {}
types = sparql.type_query(g, rootElement)
for type in types:
signposts.append(Signpost(LinkRel.type, str(type[0])))
authors = sparql.author_query(g, rootElement)
for author in authors:
author = self.select_url(author)
if author:
signposts.append(Signpost(LinkRel.author, author))
license = next(iter(sparql.license_query(g, rootElement)), [])
license = self.select_url(license)
if license:
signposts.append(Signpost(LinkRel.license, license))
citations = sparql.cite_query(g, rootElement)
for citation in citations:
citation = self.select_url(citation)
if citation:
signposts.append(Signpost(LinkRel.cite_as, citation))
sameas = sparql.sameas_query(g, rootElement)
for sa in sameas:
sa_media_type = sa[-1]
sa = self.select_url(sa[:-1])
if sa:
signposts.append(Signpost(LinkRel.describedby, sa, sa_media_type))
items = sparql.item_query(g, rootElement)
for item in items:
item_media_type = item[-1]
item = self.select_url(item[:-1])
if item:
signposts.append(Signpost(LinkRel.item, item, item_media_type))
return signposts
def process_response(
self, request: HttpRequest, response: HttpResponse
) -> HttpResponse:
if not getattr(settings, "SIGNPOSTING_PARSE_JSONLD", True):
return response
if response.get("Content-Type", "").startswith("text/html"):
soup = BeautifulSoup(response.content, "html.parser")
for script in soup.find_all("script", type="application/ld+json"):
try:
jsonld = json.loads(script.string)
signposts = self._jsonld_to_signposts(jsonld)
response._signposts = signposts
except json.JSONDecodeError as e:
print(e)
continue
return response

View File

@@ -0,0 +1,136 @@
from rdflib import Graph
from rdflib.query import Result
def root_element_query(g: Graph) -> Result:
return g.query("""
PREFIX schema: <http://schema.org/>
SELECT DISTINCT ?rootElement
WHERE {
# has a type
?rootElement a ?type ;
# has a license or an author or a creator
(schema:license | schema:author | schema:creator ) ?value .
# No incoming edges for ?rootElement
FILTER NOT EXISTS { ?s ?p ?rootElement }
}
LIMIT 1
""")
def type_query(g: Graph, rootElement: str) -> Graph:
return g.query(
"""
PREFIX schema: <?context>
SELECT DISTINCT ?type
WHERE {
?rootElement a ?type .
}
""",
initBindings={"rootElement": rootElement},
)
def author_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?author_id ?identifier ?url
WHERE {
?rootElement schema:author ?author .
BIND(?author AS ?author_id) # Get the @id of the author
OPTIONAL { ?author schema:identifier ?identifier }
OPTIONAL { ?author schema:url ?url }
}
LIMIT 1
""",
initBindings={"rootElement": rootElement},
)
def license_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?element_id ?identifier ?url
WHERE {
?rootElement schema:license ?element .
BIND(?element AS ?element_id) # Get the @id of the element
OPTIONAL { ?element schema:identifier ?identifier }
OPTIONAL { ?element schema:url ?url }
}
LIMIT 1
""",
initBindings={"rootElement": rootElement},
)
def cite_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?element_id ?identifier ?url
WHERE {
?rootElement schema:url ?element .
BIND(?element AS ?element_id) # Get the @id of the element
OPTIONAL { ?element schema:identifier ?identifier }
OPTIONAL { ?element schema:url ?url }
}
LIMIT 1
""",
initBindings={"rootElement": rootElement},
)
def sameas_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?element_id ?contentUrl ?identifier ?url ?encoding
WHERE {
?rootElement schema:sameAs ?element .
BIND(?element AS ?element_id) # Get the @id of the element
OPTIONAL { ?element schema:contentUrl ?contentUrl }
OPTIONAL { ?element schema:identifier ?identifier }
OPTIONAL { ?element schema:url ?url }
OPTIONAL { ?element schema:encodingFormat ?encoding }
}
""",
initBindings={"rootElement": rootElement},
)
def item_query(g: Graph, rootElement: str):
return g.query(
"""
PREFIX schema: <http://schema.org/>
SELECT ?element_id ?identifier ?url ?contentUrl ?encoding
WHERE {
?rootElement schema:hasPart ?element .
#VALUES ?element_type { schema:MediaObject schema:Dataset }
#?element a ?element_type .
BIND(?element AS ?element_id) # Get the @id of the element
OPTIONAL { ?element schema:contentUrl ?contentUrl }
OPTIONAL { ?element schema:identifier ?identifier }
OPTIONAL { ?element schema:url ?url }
OPTIONAL { ?element schema:encodingFormat ?encoding }
}
""",
initBindings={"rootElement": rootElement},
)

View File

@@ -13,5 +13,5 @@ def add_signposts(response: HttpResponse, *args: Signpost):
response._signposts = []
for signpost in args:
if not signpost in response._signposts:
if signpost not in response._signposts:
response._signposts.append(signpost)

View File

@@ -37,6 +37,7 @@ INSTALLED_APPS = [
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django_json_ld',
]
MIDDLEWARE = [
@@ -48,6 +49,7 @@ MIDDLEWARE = [
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django_signposting.middleware.SignpostingMiddleware',
'django_signposting.middleware.JsonLdSignpostingParserMiddleware',
]
ROOT_URLCONF = 'example.urls'
@@ -55,7 +57,9 @@ ROOT_URLCONF = 'example.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'DIRS': [
BASE_DIR / 'example/templates',
],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [

View File

@@ -0,0 +1,11 @@
<!DOCTYPE html>
<html>
<head>
{% load render_json_ld from json_ld %}
{% render_json_ld sd %}
</head>
<body>
<p>Hello, world!</p>
</body>
</html>

View File

@@ -14,11 +14,11 @@ Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from . import views
urlpatterns = [
path("", views.my_view),
path("", views.SimpleView.as_view()),
path("jsonld", views.JsonLdView.as_view()),
]

View File

@@ -1,12 +1,70 @@
from django.http import HttpResponse
from django.views import View
from django.shortcuts import render
from django_json_ld.views import JsonLdContextMixin
from django_signposting.utils import add_signposts
def my_view(request):
response = HttpResponse("Hello, world!")
from signposting import Signpost, LinkRel
# Add signpostings as string
add_signposts(response,
type="https://schema.org/Dataset",
author="https://orcid.org/0000-0001-9447-460X")
return response
class SimpleView(View):
def get(self, request):
response = HttpResponse("Hello, world!")
# Add signpostings as string
add_signposts(
response,
Signpost(LinkRel.type, "http://schema.org/Dataset"),
Signpost(LinkRel.author, "https://orcid.org/0000-0001-9447-460X")
)
return response
class JsonLdView(JsonLdContextMixin, View):
sd = {
"@context": "https://schema.org",
"@type": ["WebSite", "Dataset"],
"name": "My Dataset",
"description": "A dataset of things.",
"url": "https://example.com",
"sameAs": [
{
"@type": "MediaObject",
"contentUrl": "https://example.com/download.zip",
"encodingFormat": "application/zip"
},
{
"@type": "MediaObject",
"contentUrl": "https://example.com/metadata.json",
}
],
"author": {
"@type": "Person",
"name": "Daniel Bauer",
"url": "https://orcid.org/0000-0001-9447-460X",
},
"license": {
"@type": "CreativeWork",
"name": "CC BY 4.0",
"url": "https://creativecommons.org/licenses/by/4.0/"
},
"hasPart": [
{
"@type": "ImageObject",
"url": "http://example.com/image.png",
"encodingFormat": "image/png"
},
{
"@type": "ImageObject",
"url": "http://example.com/image2.png",
"encodingFormat": "image/png"
}
]
}
def get(self, request):
return render(request, "jsonld.html", context={"sd": self.sd})

17
example/requirements.txt Normal file
View File

@@ -0,0 +1,17 @@
asgiref==3.8.1
beautifulsoup4==4.12.3
certifi==2024.8.30
charset-normalizer==3.4.0
django==5.1.3
django-json-ld==0.0.5
django-signposting==0.10.0
httplink==0.2.0
idna==3.10
pyparsing==3.2.0
rdflib==7.1.1
requests==2.32.3
rfc3987==1.3.8
signposting==0.9.9
soupsieve==2.6
sqlparse==0.5.1
urllib3==2.2.3

View File

@@ -8,12 +8,14 @@ authors = [
{name = "Daniel Bauer", email = "github@dbauer.me"}
]
description = "Add FAIR signpostings to response headers in Django"
version = "0.10.0"
version = "0.10.1"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"Django>=3.0",
"beautifulsoup4>=4.12.3",
"django>=3.0",
"signposting>=0.9.9",
"rdflib>=7.1.1",
]
license = {file= "LICENSE"}

View File

@@ -1,4 +1,3 @@
import pytest
from django.conf import settings

View File

@@ -0,0 +1,220 @@
import json
from django.http import HttpRequest, HttpResponse
from django_signposting.middleware import JsonLdSignpostingParserMiddleware
from signposting import Signpost, LinkRel
def jsonld_test_runner(jsonld, expected_signposts):
response = HttpResponse(f"""
<html><head>
<script type="application/ld+json">{json.dumps(jsonld)}</script>
</head>
<body></body>
</html>
""")
response.status_code = 200
middleware = JsonLdSignpostingParserMiddleware(lambda request: response)
response = middleware(HttpRequest())
for expected_signpost in expected_signposts:
assert expected_signpost in response._signposts
assert len(expected_signposts) == len(response._signposts)
def test_jsonld_signposting_basic():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@type": "WebPage",
"author": {"url": "http://example.com/author"},
"url": "http://example.com/url",
"license": {"identifier": "http://example.com/license"},
"hasPart": [
{
"url": "http://example.com/image1.jpg",
},
{
"url": "http://example.com/image2.jpg",
},
],
"sameAs": [
{
"url": "http://example.com/metadata.json",
},
{
"url": "http://example.com/metadata.xml",
},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/author"),
Signpost(LinkRel.cite_as, "http://example.com/url"),
Signpost(LinkRel.license, "http://example.com/license"),
Signpost(LinkRel.item, "http://example.com/image1.jpg"),
Signpost(LinkRel.item, "http://example.com/image2.jpg"),
Signpost(LinkRel.describedby, "http://example.com/metadata.json"),
Signpost(LinkRel.describedby, "http://example.com/metadata.xml"),
],
)
def test_jsonld_empty_signposting():
jsonld_test_runner({}, [])
def test_jsonld_signposting_media_types():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@type": "WebPage",
"author": {"url": "http://example.com/author"},
"url": "http://example.com/url",
"license": {"identifier": "http://example.com/license"},
"hasPart": [
{
"url": "http://example.com/image1.jpg",
"encodingFormat": "image/jpeg",
},
{"url": "http://example.com/image2.jpg", "encodingFormat": "image/png"},
],
"sameAs": [
{
"url": "http://example.com/metadata.json",
"encodingFormat": "application/json",
},
{
"url": "http://example.com/metadata.xml",
"encodingFormat": "application/xml",
},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/author"),
Signpost(LinkRel.cite_as, "http://example.com/url"),
Signpost(LinkRel.license, "http://example.com/license"),
Signpost(LinkRel.item, "http://example.com/image1.jpg", "image/jpeg"),
Signpost(LinkRel.item, "http://example.com/image2.jpg", "image/png"),
Signpost(
LinkRel.describedby,
"http://example.com/metadata.json",
"application/json",
),
Signpost(
LinkRel.describedby,
"http://example.com/metadata.xml",
"application/xml",
),
],
)
def test_jsonld_signposting_property_precedence():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@type": "WebPage",
"author": {
"identifier": "http://example.com/author-ident",
"url": "http://example.com/author-url",
},
"hasPart": [
{
"identifier": "http://example.com/image1-identifier.jpg",
"url": "http://example.com/image1-url.jpg",
"contentUrl": "http://example.com/image1-content.jpg",
},
{
"identifier": "http://example.com/image2-identifier.jpg",
"url": "http://example.com/image2-url.jpg",
},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/author-url"),
Signpost(LinkRel.item, "http://example.com/image1-content.jpg"),
Signpost(LinkRel.item, "http://example.com/image2-url.jpg"),
],
)
def test_jsonld_signposting_ids():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@graph": [
{
"@type": "WebPage",
"author": {"@id": "http://example.com/author"},
"url": "http://example.com/url",
"license": {"@id": "http://example.com/license"},
"hasPart": [
{
"@id": "http://example.com/image1.jpg",
},
],
"sameAs": [
{
"@id": "http://example.com/metadata.json",
},
],
},
{"@id": "http://example.com/author", "@type": "Person"},
{"@id": "http://example.com/license"},
{
"@id": "http://example.com/image1.jpg",
"encodingFormat": "image/jpeg",
},
{"@id": "http://example.com/metadata.json"},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/author"),
Signpost(LinkRel.cite_as, "http://example.com/url"),
Signpost(LinkRel.license, "http://example.com/license"),
Signpost(LinkRel.item, "http://example.com/image1.jpg", "image/jpeg"),
Signpost(LinkRel.describedby, "http://example.com/metadata.json"),
],
)
def test_jsonld_signposting_ids_with_url():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@graph": [
{
"@type": "WebPage",
"author": {"@id": "http://example.com/author"},
},
{
"@id": "http://example.com/author",
"url": "http://example.com/realAuthorUrl",
},
],
},
[
Signpost(LinkRel.type, "http://schema.org/WebPage"),
Signpost(LinkRel.author, "http://example.com/realAuthorUrl"),
],
)
def test_jsonld_signposting_multiple_types():
jsonld_test_runner(
{
"@context": "http://schema.org",
"@type": "Dataset",
"author": "http://example.com/author",
},
[
Signpost(LinkRel.type, "http://schema.org/Dataset"),
Signpost(LinkRel.author, "http://example.com/author"),
],
)

View File

@@ -1,8 +1,8 @@
from re import A
from django.http import HttpResponse
from django_signposting.middleware import SignpostingMiddleware
from signposting import LinkRel, Signpost
def test_middleware_no_signposting():
response = HttpResponse()
response.status_code = 200

View File

@@ -5,7 +5,7 @@ from signposting import Signpost, LinkRel
def test_add_signpost():
response = HttpResponse()
add_signposts(response, Signpost(LinkRel.item,"http://example.com"))
add_signposts(response, Signpost(LinkRel.item, "http://example.com"))
assert len(response._signposts) == 1
@@ -18,7 +18,6 @@ def test_add_multiple_signposts():
Signpost(LinkRel.author, "https://example3.com"),
)
assert len(response._signposts) == 3