emby-mcp/internal/emby/spec_conformance_test.go

package emby

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"testing"

	"gopkg.in/yaml.v3"
)

// This test intentionally keeps a small, local OpenAPI model. The manifest is
// the hand-maintained record of the upstream calls made by internal/emby; it
// is not generated from the Go AST.
type specDoc struct {
	Paths      map[string]specPathItem `yaml:"paths"`
	Components struct {
		Schemas map[string]specSchema `yaml:"schemas"`
	} `yaml:"components"`
}

// Path items also contain summary/description keys, which are not operations.
// Decode only the HTTP method children into the operation map.
type specPathItem map[string]specOp

func (p *specPathItem) UnmarshalYAML(node *yaml.Node) error {
	var raw map[string]yaml.Node
	if err := node.Decode(&raw); err != nil {
		return err
	}
	result := make(specPathItem)
	for method, child := range raw {
		switch method {
		case "get", "post", "put", "patch", "delete", "head", "options", "trace":
			var op specOp
			if err := child.Decode(&op); err != nil {
				return err
			}
			result[method] = op
		}
	}
	*p = result
	return nil
}

type specOp struct {
	OperationID string                  `yaml:"operationId"`
	Parameters  []specParam             `yaml:"parameters"`
	RequestBody *specRequestBody        `yaml:"requestBody"`
	Responses   map[string]specResponse `yaml:"responses"`
}

type specParam struct {
	Name     string     `yaml:"name"`
	In       string     `yaml:"in"`
	Required bool       `yaml:"required"`
	Schema   specSchema `yaml:"schema"`
}

type specRequestBody struct {
	Content  map[string]specMedia `yaml:"content"`
	Required bool                 `yaml:"required"`
}

type specResponse struct {
	Content map[string]specMedia `yaml:"content"`
}

type specMedia struct {
	Schema specSchema `yaml:"schema"`
}

type specSchema struct {
	Ref        string                `yaml:"$ref"`
	Type       string                `yaml:"type"`
	Enum       []string              `yaml:"enum"`
	Properties map[string]specSchema `yaml:"properties"`
	Items      *specSchema           `yaml:"items"`
}

func (d *specDoc) resolve(s specSchema) specSchema {
	if s.Ref == "" {
		return s
	}
	const prefix = "#/components/schemas/"
	name := strings.TrimPrefix(s.Ref, prefix)
	if name == s.Ref {
		return s
	}
	resolved, ok := d.Components.Schemas[name]
	if !ok {
		return s
	}
	return d.resolve(resolved)
}

// hasProperty accepts paths into array responses, so Items.Name means the
// Name property of each item in a QueryResult or top-level array response.
func (d *specDoc) hasProperty(s specSchema, dotted string) bool {
	current := s
	for _, part := range strings.Split(dotted, ".") {
		current = d.resolve(current)
		for current.Type == "array" && current.Items != nil {
			current = d.resolve(*current.Items)
		}
		property, ok := current.Properties[part]
		if !ok {
			return false
		}
		current = property
	}
	return true
}

func (d *specDoc) schemaProperties(s specSchema) map[string]specSchema {
	return d.resolve(s).Properties
}

func (d *specDoc) requestSchema(op specOp) (specSchema, bool) {
	if op.RequestBody == nil {
		return specSchema{}, false
	}
	media, ok := op.RequestBody.Content["application/json"]
	return media.Schema, ok
}

func (d *specDoc) responseSchema(op specOp) (specSchema, bool) {
	response, ok := op.Responses["200"]
	if !ok {
		return specSchema{}, false
	}
	media, ok := response.Content["application/json"]
	return media.Schema, ok
}

type endpointUsage struct {
	tool       string
	callSite   string
	method     string
	specPath   string
	query      []string
	path       []string
	bodyFields []string
	respFields []string
	enumUse    map[string][]string
}

// endpointUsages has one row for each upstream call site, not merely one row
// for each unique URL. Keep this table beside the client so a new call site
// requires an explicit conformance review.
// mediaItemRespFields is the full response-field set consumed via
// toMediaItem: the item fields plus stream and per-user watch-state fields.
var mediaItemRespFields = []string{
	"Items", "TotalRecordCount",
	"Items.Name", "Items.Artists", "Items.Album", "Items.AlbumId",
	"Items.AlbumArtist", "Items.ParentIndexNumber", "Items.IndexNumber",
	"Items.DateCreated", "Items.PremiereDate", "Items.ProductionYear",
	"Items.Genres", "Items.Overview", "Items.MediaType", "Items.RunTimeTicks",
	"Items.Bitrate", "Items.Id", "Items.Path", "Items.Type",
	"Items.SeriesName", "Items.SeriesId", "Items.SeasonId", "Items.SeasonName",
	"Items.LocationType",
	"Items.UserData.Played", "Items.UserData.PlayedPercentage",
	"Items.UserData.PlaybackPositionTicks",
	"Items.MediaSources.MediaStreams.Index", "Items.MediaSources.MediaStreams.Type",
	"Items.MediaSources.MediaStreams.Language", "Items.MediaSources.MediaStreams.Codec",
	"Items.MediaSources.MediaStreams.Title", "Items.MediaSources.MediaStreams.DisplayTitle",
	"Items.MediaSources.MediaStreams.IsDefault", "Items.MediaSources.MediaStreams.IsForced",
	"Items.MediaSources.MediaStreams.IsExternal",
	"Items.MediaSources.MediaStreams.IsTextSubtitleStream",
	"Items.MediaSources.MediaStreams.ExtraData",
}

var endpointUsages = []endpointUsage{
	{tool: "startup auth", callSite: "library.go:19", method: "post", specPath: "/Users/AuthenticateByName", bodyFields: []string{"Username", "Pw"}, respFields: []string{"User.Id", "User.Name", "AccessToken", "ServerId"}},
	{tool: "shutdown", callSite: "library.go:33", method: "post", specPath: "/Sessions/Logout"},
	{tool: "retrieve_user_list", callSite: "users.go:20", method: "get", specPath: "/Users/{Id}", path: []string{"Id"}, respFields: []string{"Id", "Name"}},
	{tool: "retrieve_user_list", callSite: "users.go:25", method: "get", specPath: "/Users/Public", respFields: []string{"Id", "Name"}},
	{tool: "retrieve_library_list, select_library", callSite: "library.go:39", method: "get", specPath: "/Library/MediaFolders", respFields: []string{"Items", "Items.Name", "Items.Type", "Items.CollectionType", "Items.Id", "TotalRecordCount"}},
	{tool: "retrieve_genre_list", callSite: "library.go:58", method: "get", specPath: "/Genres", query: []string{"Recursive", "ParentId"}, respFields: []string{"Items", "Items.Name", "TotalRecordCount"}},
	{tool: "search_for_item, retrieve_next_episode (series resolve)", callSite: "items.go:135", method: "get", specPath: "/Users/{UserId}/Items", path: []string{"UserId"}, query: []string{"Recursive", "MediaTypes", "Fields", "ParentId", "SearchTerm", "Artists", "Genres", "Years", "MinStartDate", "MaxEndDate", "Limit", "Filters", "IncludeItemTypes", "SortBy", "SortOrder", "EnableUserData"}, respFields: mediaItemRespFields},
	{tool: "retrieve_item_children", callSite: "shows.go:28", method: "get", specPath: "/Users/{UserId}/Items", path: []string{"UserId"}, query: []string{"Recursive", "ParentId", "Fields", "SortBy"}, respFields: mediaItemRespFields},
	{tool: "retrieve_season_list", callSite: "shows.go:48", method: "get", specPath: "/Shows/{Id}/Seasons", path: []string{"Id"}, query: []string{"UserId", "Fields"}, respFields: mediaItemRespFields},
	{tool: "retrieve_episode_list", callSite: "shows.go:75", method: "get", specPath: "/Shows/{Id}/Episodes", path: []string{"Id"}, query: []string{"UserId", "SeasonId", "Fields", "SortBy", "EnableUserData"}, respFields: mediaItemRespFields},
	{tool: "retrieve_next_episode (next_unplayed)", callSite: "shows.go:97", method: "get", specPath: "/Shows/NextUp", query: []string{"UserId", "SeriesId", "Fields", "EnableUserData", "Limit"}, respFields: mediaItemRespFields},
	{tool: "retrieve_next_episode (latest)", callSite: "shows.go:120", method: "get", specPath: "/Shows/{Id}/Episodes", path: []string{"Id"}, query: []string{"UserId", "Fields", "SortBy", "SortOrder", "Limit", "EnableUserData"}, respFields: mediaItemRespFields},
	{tool: "retrieve_playlist_list, create_playlist, modify_playlist_name", callSite: "playlists.go:76", method: "get", specPath: "/Users/{UserId}/Items", path: []string{"UserId"}, query: []string{"ParentId", "Recursive", "Fields", "Ids"}, respFields: []string{"Items", "Items.Name", "Items.Overview", "Items.Genres", "Items.RunTimeTicks", "Items.Type", "Items.Id", "Items.DateCreated", "TotalRecordCount"}},
	{tool: "retrieve_playlist_list", callSite: "playlists.go:99", method: "get", specPath: "/Users/ItemAccess", query: []string{"ItemId"}, respFields: []string{"Items", "Items.Id", "Items.Name", "Items.UserItemShareLevel", "TotalRecordCount"}},
	{tool: "retrieve_playlist_items", callSite: "playlists.go:120", method: "get", specPath: "/Playlists/{Id}/Items", path: []string{"Id"}, query: []string{"UserId", "Fields"}, respFields: append(append([]string{}, mediaItemRespFields...), "Items.PlaylistItemId")},
	{tool: "create_playlist", callSite: "playlists.go:175", method: "post", specPath: "/Playlists", query: []string{"Name", "MediaType"}, respFields: []string{"Id"}},
	{tool: "modify_playlist_name, create_playlist(overview)", callSite: "playlists.go:211", method: "get", specPath: "/Users/{UserId}/Items/{Id}", path: []string{"UserId", "Id"}, respFields: []string{"Name", "Overview"}},
	{tool: "modify_playlist_name, create_playlist(overview)", callSite: "playlists.go:223", method: "post", specPath: "/Items/{ItemId}", path: []string{"ItemId"}, bodyFields: []string{"Name", "Overview"}},
	{tool: "add_items_to_playlist, create_playlist", callSite: "playlists.go:230", method: "post", specPath: "/Playlists/{Id}/Items", path: []string{"Id"}, query: []string{"Ids", "UserId"}, respFields: []string{"ItemAddedCount"}},
	{tool: "remove_items_from_playlist", callSite: "playlists.go:242", method: "post", specPath: "/Playlists/{Id}/Items/Delete", path: []string{"Id"}, query: []string{"EntryIds"}},
	{tool: "reorder_items_on_playlist", callSite: "playlists.go:248", method: "post", specPath: "/Playlists/{Id}/Items/{ItemId}/Move/{NewIndex}", path: []string{"Id", "ItemId", "NewIndex"}},
	{tool: "share_playlist_public", callSite: "playlists.go:257", method: "post", specPath: "/Items/{Id}/MakePublic", path: []string{"Id"}},
	{tool: "stop_sharing_playlist", callSite: "playlists.go:259", method: "post", specPath: "/Items/{Id}/MakePrivate", path: []string{"Id"}},
	{tool: "share_playlist_user_access", callSite: "playlists.go:269", method: "post", specPath: "/Items/Access", bodyFields: []string{"ItemIds", "UserIds", "ItemAccess"}, enumUse: map[string][]string{"UserItemShareLevel": {"None", "Read", "Write", "Manage", "ManageDelete"}}},
	{tool: "retrieve_player_list", callSite: "sessions.go:41", method: "get", specPath: "/Sessions", query: []string{"ControllableByUserId"}, respFields: []string{"Client", "Id", "DeviceId", "DeviceName", "RemoteEndPoint", "PlayableMediaTypes", "NowPlayingItem.Name", "NowPlayingItem.Artists", "NowPlayingItem.Album", "NowPlayingItem.IndexNumber", "NowPlayingItem.ParentIndexNumber", "NowPlayingItem.Id", "NowPlayingItem.RunTimeTicks", "PlayState.PositionTicks", "PlayState.IsPaused"}},
	{tool: "retrieve_player_queue", callSite: "sessions.go:121", method: "get", specPath: "/Sessions/PlayQueue", query: []string{"Id"}, respFields: []string{"Items", "Items.Name", "Items.Artists", "Items.Album", "Items.AlbumId", "Items.AlbumArtist", "Items.ParentIndexNumber", "Items.IndexNumber", "Items.DateCreated", "Items.PremiereDate", "Items.ProductionYear", "Items.Genres", "Items.Overview", "Items.MediaType", "Items.Bitrate", "Items.RunTimeTicks", "Items.Id", "Items.PlaylistItemId", "TotalRecordCount"}},
	{tool: "control_media_player (PlayNow)", callSite: "sessions.go:260", method: "post", specPath: "/Sessions/{Id}/Playing", path: []string{"Id"}, query: []string{"ItemIds", "PlayCommand"}, bodyFields: []string{"PlayCommand", "ControllingUserId"}, enumUse: map[string][]string{"PlayCommand": {"PlayNow"}}},
	{tool: "control_media_player (other commands)", callSite: "sessions.go:281", method: "post", specPath: "/Sessions/{Id}/Playing/{Command}", path: []string{"Id", "Command"}, bodyFields: []string{"Command", "SeekPositionTicks", "ControllingUserId"}, enumUse: map[string][]string{"PlaystateCommand": {"Stop", "Pause", "Unpause", "NextTrack", "PreviousTrack", "Seek", "Rewind", "FastForward", "PlayPause", "SeekRelative"}}},
	{tool: "retrieve_now_playing, set_subtitle, set_audio_track", callSite: "sessions.go:183", method: "get", specPath: "/Sessions", respFields: []string{"Id", "SupportedCommands", "NowPlayingItem.Id", "NowPlayingItem.Name", "NowPlayingItem.SeriesName", "NowPlayingItem.MediaType", "NowPlayingItem.MediaSources.MediaStreams.Index", "NowPlayingItem.MediaSources.MediaStreams.Type", "NowPlayingItem.MediaSources.MediaStreams.Language", "NowPlayingItem.MediaSources.MediaStreams.Codec", "NowPlayingItem.MediaSources.MediaStreams.Title", "NowPlayingItem.MediaSources.MediaStreams.DisplayTitle", "NowPlayingItem.MediaSources.MediaStreams.IsDefault", "NowPlayingItem.MediaSources.MediaStreams.IsForced", "NowPlayingItem.MediaSources.MediaStreams.IsExternal", "PlayState.PositionTicks", "PlayState.IsPaused", "PlayState.CanSeek", "PlayState.AudioStreamIndex", "PlayState.SubtitleStreamIndex"}},
	{tool: "set_subtitle, set_audio_track", callSite: "sessions.go:202", method: "post", specPath: "/Sessions/{Id}/Command", path: []string{"Id"}, bodyFields: []string{"Name", "ControllingUserId", "Arguments"}},
	{tool: "set_subtitle, set_audio_track (restart fallback)", callSite: "sessions.go:227", method: "post", specPath: "/Sessions/{Id}/Playing", path: []string{"Id"}, query: []string{"ItemIds", "PlayCommand"}, bodyFields: []string{"PlayCommand", "ControllingUserId", "SubtitleStreamIndex", "AudioStreamIndex", "StartPositionTicks"}, enumUse: map[string][]string{"PlayCommand": {"PlayNow"}}},
	{tool: "retrieve_now_playing, set_subtitle, set_audio_track", callSite: "items.go:165", method: "get", specPath: "/Users/{UserId}/Items/{Id}", path: []string{"UserId", "Id"}, query: []string{"Fields", "EnableUserData"}, respFields: []string{"Id", "Name", "SeriesName", "MediaType", "MediaSources.MediaStreams.Index", "MediaSources.MediaStreams.Type", "MediaSources.MediaStreams.Language", "MediaSources.MediaStreams.Codec", "MediaSources.MediaStreams.Title", "MediaSources.MediaStreams.DisplayTitle", "MediaSources.MediaStreams.IsDefault", "MediaSources.MediaStreams.IsForced", "MediaSources.MediaStreams.IsExternal", "MediaSources.MediaStreams.IsTextSubtitleStream", "MediaSources.MediaStreams.ExtraData"}},
	{tool: "http auth (token→user)", callSite: "sessions.go:121", method: "get", specPath: "/Sessions", respFields: []string{"DeviceId", "UserId"}},
	{tool: "http auth (X-Emby-User-Id)", callSite: "users.go:42", method: "get", specPath: "/Users/{Id}", path: []string{"Id"}, respFields: []string{"Id", "Name"}},
	{tool: "startup check (http)", callSite: "system.go:16", method: "get", specPath: "/System/Info/Public", respFields: []string{"Id", "ServerName", "Version"}},
}

type conformanceViolation struct {
	usage     endpointUsage
	operation string
	check     string
	detail    string
}

type knownIssue struct {
	id          string
	kind        string
	description string
	evidence    string
	action      string
	holds       func(*specDoc) bool
	matches     func(conformanceViolation) bool
	applies     func(endpointUsage) bool
}

var knownIssues = []knownIssue{
	{
		id:          "SPEC-GAP-ITEMACCESS-ITEMID",
		kind:        "spec-gap",
		description: "GET /Users/ItemAccess accepts ItemId, but the OpenAPI operation does not declare it.",
		evidence:    "Emby.MCP/hotfixes/emby/user_service_api.py adds item_id to this exact request.",
		action:      "Allowlist the call; keep the client behavior unchanged.",
		holds: func(d *specDoc) bool {
			return !hasParameter(d.Paths["/Users/ItemAccess"]["get"], "ItemId", "query")
		},
		applies: func(u endpointUsage) bool { return u.callSite == "playlists.go:99" },
		matches: func(v conformanceViolation) bool {
			return v.usage.callSite == "playlists.go:99" && v.check == "parameter" && strings.Contains(v.detail, "ItemId")
		},
	},
	{
		id:          "CODE-DEVIATION-PLAYREQUEST-PLAYCOMMAND",
		kind:        "code-deviation",
		description: "PlayNow includes PlayCommand in the JSON PlayRequest body although the schema omits that property.",
		evidence:    "internal/emby/sessions.go sends playRequest{PlayCommand, ...}; the spec PlayRequest has no PlayCommand.",
		action:      "Report only; a later client cleanup can remove PlayCommand from the body struct.",
		holds: func(d *specDoc) bool {
			op := d.Paths["/Sessions/{Id}/Playing"]["post"]
			schema, ok := d.requestSchema(op)
			return ok && !hasSchemaProperty(d, schema, "PlayCommand")
		},
		applies: func(u endpointUsage) bool {
			return u.callSite == "sessions.go:260" || u.callSite == "sessions.go:227"
		},
		matches: func(v conformanceViolation) bool {
			return (v.usage.callSite == "sessions.go:260" || v.usage.callSite == "sessions.go:227") &&
				v.check == "body" && strings.Contains(v.detail, "PlayCommand")
		},
	},
	{
		id:          "SPEC-GAP-PLAYREQUEST-STARTPOSITIONTICKS",
		kind:        "spec-gap",
		description: "PlayNow-with-options sends StartPositionTicks in the PlayRequest body, which the OpenAPI schema omits; real Emby accepts it for resume-position restarts.",
		evidence:    "internal/emby/sessions.go PlayNowWithOptions sends playRequest{StartPositionTicks}; used by set_subtitle/set_audio_track restart fallback.",
		action:      "Allowlist the field; keep the client behavior unchanged.",
		holds: func(d *specDoc) bool {
			op := d.Paths["/Sessions/{Id}/Playing"]["post"]
			schema, ok := d.requestSchema(op)
			return ok && !hasSchemaProperty(d, schema, "StartPositionTicks")
		},
		applies: func(u endpointUsage) bool { return u.callSite == "sessions.go:227" },
		matches: func(v conformanceViolation) bool {
			return v.usage.callSite == "sessions.go:227" && v.check == "body" && strings.Contains(v.detail, "StartPositionTicks")
		},
	},
	{
		id:          "SPEC-QUIRK-MEDIASTREAM-EXTRADATA",
		kind:        "spec-quirk",
		description: "MediaStream spells the lyrics payload field Extradata, while Emby JSON uses ExtraData.",
		evidence:    "Go JSON decoding is case-insensitive, so the client's ExtraData field still reads the emitted value.",
		action:      "Document only; do not patch the checked-in upstream spec in this work.",
		holds: func(d *specDoc) bool {
			props := d.schemaProperties(d.Components.Schemas["MediaStream"])
			_, oldName := props["Extradata"]
			_, newName := props["ExtraData"]
			return oldName && !newName
		},
		applies: func(u endpointUsage) bool {
			switch u.callSite {
			case "items.go:135", "items.go:165", "playlists.go:120",
				"shows.go:28", "shows.go:48", "shows.go:75",
				"shows.go:97", "shows.go:120":
				return true
			}
			return false
		},
		matches: func(v conformanceViolation) bool {
			return v.check == "response field" && strings.Contains(v.detail, "ExtraData")
		},
	},
	{
		id:          "SPEC-GAP-EPISODES-RESPONSE-SCHEMA",
		kind:        "spec-gap",
		description: "GET /Shows/{Id}/Episodes declares no JSON 200-response schema, so episode list responses cannot be field-checked.",
		evidence:    "The spec's 200 response has no application/json schema; real Emby returns QueryResult_BaseItemDto like the sibling /Shows/{Id}/Seasons operation.",
		action:      "Allowlist the missing schema; respFields on these rows are unchecked.",
		holds: func(d *specDoc) bool {
			op := d.Paths["/Shows/{Id}/Episodes"]["get"]
			_, ok := d.responseSchema(op)
			return !ok
		},
		applies: func(u endpointUsage) bool {
			return u.callSite == "shows.go:75" || u.callSite == "shows.go:120"
		},
		matches: func(v conformanceViolation) bool {
			return v.check == "response" && strings.Contains(v.detail, "no JSON 200-response schema") &&
				(v.usage.callSite == "shows.go:75" || v.usage.callSite == "shows.go:120")
		},
	},
	{
		id:          "SPEC-GAP-ITEMBYID-USERDATA-PARAMS",
		kind:        "spec-gap",
		description: "GET /Users/{UserId}/Items/{Id} does not declare the Fields or EnableUserData query parameters.",
		evidence:    "Real Emby accepts both; used by GetItem for stream/userdata fetch in retrieve_now_playing.",
		action:      "Allowlist the parameters; keep the client behavior unchanged.",
		holds: func(d *specDoc) bool {
			op := d.Paths["/Users/{UserId}/Items/{Id}"]["get"]
			return !hasParameter(op, "Fields", "query") || !hasParameter(op, "EnableUserData", "query")
		},
		applies: func(u endpointUsage) bool { return u.callSite == "items.go:165" },
		matches: func(v conformanceViolation) bool {
			return v.usage.callSite == "items.go:165" && v.check == "parameter" &&
				(strings.Contains(v.detail, "\"Fields\"") || strings.Contains(v.detail, "\"EnableUserData\""))
		},
	},
	{
		id:          "SPEC-QUIRK-PLAYING-ITEMIDS-TYPE",
		kind:        "spec-quirk",
		description: "The PlayNow ItemIds parameter is typed array<int64>, although real item IDs are strings.",
		evidence:    "The spec description says comma-delimited and the client sends comma-separated string IDs, matching Emby behavior.",
		action:      "Document only; retain the string query encoding.",
		holds: func(d *specDoc) bool {
			op := d.Paths["/Sessions/{Id}/Playing"]["post"]
			for _, p := range op.Parameters {
				if p.Name == "ItemIds" && p.In == "query" {
					s := d.resolve(p.Schema)
					return s.Type == "array" && s.Items != nil && d.resolve(*s.Items).Type == "integer"
				}
			}
			return false
		},
		applies: func(u endpointUsage) bool {
			return u.callSite == "sessions.go:260" || u.callSite == "sessions.go:227"
		},
	},
}

func hasParameter(op specOp, name, in string) bool {
	for _, p := range op.Parameters {
		if p.Name == name && p.In == in {
			return true
		}
	}
	return false
}

func hasSchemaProperty(d *specDoc, schema specSchema, name string) bool {
	_, ok := d.schemaProperties(schema)[name]
	return ok
}

func loadEmbySpec(t *testing.T) *specDoc {
	t.Helper()
	_, sourceFile, _, ok := runtime.Caller(0)
	if !ok {
		t.Fatal("could not locate spec conformance test")
	}
	specPath := filepath.Join(filepath.Dir(sourceFile), "..", "..", "docs", "api", "emby_openapi.yaml")
	data, err := os.ReadFile(specPath)
	if err != nil {
		t.Fatalf("read OpenAPI spec %s: %v", specPath, err)
	}
	var doc specDoc
	if err := yaml.Unmarshal(data, &doc); err != nil {
		t.Fatalf("parse OpenAPI spec %s: %v", specPath, err)
	}
	return &doc
}

func TestSpecConformance(t *testing.T) {
	doc := loadEmbySpec(t)
	var violations []conformanceViolation
	byUsage := make(map[string][]conformanceViolation)

	addViolation := func(usage endpointUsage, operation, check, detail string) {
		v := conformanceViolation{usage: usage, operation: operation, check: check, detail: detail}
		violations = append(violations, v)
		key := usage.callSite + " " + usage.method + " " + usage.specPath
		byUsage[key] = append(byUsage[key], v)
	}

	for _, usage := range endpointUsages {
		op, pathExists := doc.Paths[usage.specPath]
		if !pathExists {
			addViolation(usage, "", "path", "spec path does not exist")
			continue
		}
		specOperation, methodExists := op[strings.ToLower(usage.method)]
		if !methodExists {
			addViolation(usage, "", "method", fmt.Sprintf("%s operation does not exist", strings.ToUpper(usage.method)))
			continue
		}
		operation := specOperation.OperationID
		for _, name := range usage.query {
			if !hasParameter(specOperation, name, "query") {
				addViolation(usage, operation, "parameter", fmt.Sprintf("query parameter %q is not declared", name))
			}
		}
		for _, name := range usage.path {
			if !hasParameter(specOperation, name, "path") {
				addViolation(usage, operation, "parameter", fmt.Sprintf("path parameter %q is not declared", name))
			}
		}
		sendQuery := make(map[string]bool, len(usage.query))
		for _, name := range usage.query {
			sendQuery[name] = true
		}
		sendPath := make(map[string]bool, len(usage.path))
		for _, name := range usage.path {
			sendPath[name] = true
		}
		for _, parameter := range specOperation.Parameters {
			if !parameter.Required || (parameter.In != "query" && parameter.In != "path") {
				continue
			}
			sent := (parameter.In == "query" && sendQuery[parameter.Name]) || (parameter.In == "path" && sendPath[parameter.Name])
			if !sent {
				addViolation(usage, operation, "required parameter", fmt.Sprintf("required %s parameter %q is not in the client send-set", parameter.In, parameter.Name))
			}
		}

		if len(usage.bodyFields) > 0 {
			schema, ok := doc.requestSchema(specOperation)
			if !ok {
				addViolation(usage, operation, "body", "client sends a body but the operation has no JSON request schema")
			} else {
				properties := doc.schemaProperties(schema)
				for _, field := range usage.bodyFields {
					if _, ok := properties[field]; !ok {
						addViolation(usage, operation, "body", fmt.Sprintf("body field %q is absent from the request schema", field))
					}
				}
			}
		}
		if len(usage.respFields) > 0 {
			schema, ok := doc.responseSchema(specOperation)
			if !ok {
				addViolation(usage, operation, "response", "client decodes a response but the operation has no JSON 200-response schema")
			} else {
				for _, field := range usage.respFields {
					if !doc.hasProperty(schema, field) {
						addViolation(usage, operation, "response field", fmt.Sprintf("response field %q is absent from the 200-response schema", field))
					}
				}
			}
		}
		for schemaName, values := range usage.enumUse {
			schema, ok := doc.Components.Schemas[schemaName]
			if !ok {
				addViolation(usage, operation, "enum", fmt.Sprintf("enum schema %q does not exist", schemaName))
				continue
			}
			allowed := make(map[string]bool, len(doc.resolve(schema).Enum))
			for _, value := range doc.resolve(schema).Enum {
				allowed[value] = true
			}
			for _, value := range values {
				if !allowed[value] {
					addViolation(usage, operation, "enum", fmt.Sprintf("value %q is absent from enum schema %q", value, schemaName))
				}
			}
		}
	}

	knownForViolation := func(v conformanceViolation) string {
		for _, issue := range knownIssues {
			if issue.matches != nil && issue.matches(v) {
				return issue.id
			}
		}
		return ""
	}
	var unexpected []conformanceViolation
	for _, v := range violations {
		if knownForViolation(v) == "" {
			unexpected = append(unexpected, v)
		}
	}
	for _, issue := range knownIssues {
		if !issue.holds(doc) {
			t.Errorf("stale known issue %s: predicate no longer holds", issue.id)
		}
		if issue.matches == nil {
			continue
		}
		matched := false
		for _, v := range violations {
			if issue.matches(v) {
				matched = true
				break
			}
		}
		if !matched {
			t.Errorf("known issue %s is no longer observed by the manifest checks", issue.id)
		}
	}

	for _, usage := range endpointUsages {
		key := usage.callSite + " " + usage.method + " " + usage.specPath
		entries := byUsage[key]
		known := make([]string, 0, len(entries))
		unexpectedCount := 0
		for _, v := range entries {
			if issueID := knownForViolation(v); issueID != "" {
				known = append(known, issueID)
			} else {
				unexpectedCount++
			}
		}
		for _, issue := range knownIssues {
			if issue.applies != nil && issue.applies(usage) && issue.holds(doc) {
				known = append(known, issue.id)
			}
		}
		status := "PASS"
		if unexpectedCount > 0 {
			status = "FAIL"
		} else if len(known) > 0 {
			status = "PASS (known issue)"
		}
		operation := ""
		if path, ok := doc.Paths[usage.specPath]; ok {
			operation = path[strings.ToLower(usage.method)].OperationID
		}
		t.Logf("%s %s %s; known=%s", status, operation, key, strings.Join(uniqueStrings(known), ","))
	}
	if len(unexpected) > 0 {
		for _, v := range unexpected {
			t.Errorf("unexpected conformance violation: %s (%s) %s: %s", v.operation, v.usage.callSite, v.check, v.detail)
		}
	}
}

func uniqueStrings(values []string) []string {
	seen := make(map[string]bool, len(values))
	result := make([]string, 0, len(values))
	for _, value := range values {
		if !seen[value] {
			seen[value] = true
			result = append(result, value)
		}
	}
	sort.Strings(result)
	return result
}