WEKO3
アイテム
{"_buckets": {"deposit": "85f3ed46-d162-42e2-b39d-30e311e59d3e"}, "_deposit": {"created_by": 23, "id": "2001350", "owners": [23], "pid": {"revision_id": 0, "type": "depid", "value": "2001350"}, "status": "published"}, "_oai": {"id": "oai:repository.nii.ac.jp:02001350", "sets": ["1697763112205"]}, "author_link": [], "control_number": "2001350", "item_30002_conference34": {"attribute_name": "会議記述", "attribute_value_mlt": [{"subitem_conference_country": "JPN", "subitem_conference_date": {"subitem_conference_date_language": "en", "subitem_conference_end_day": "15", "subitem_conference_end_month": "12", "subitem_conference_end_year": "2023", "subitem_conference_period": "December 12-15, 2023", "subitem_conference_start_day": "12", "subitem_conference_start_month": "12", "subitem_conference_start_year": "2023"}, "subitem_conference_names": [{"subitem_conference_name": "NTCIR-17 Conference", "subitem_conference_name_language": "en"}], "subitem_conference_sequence": "17", "subitem_conference_sponsors": [{"subitem_conference_sponsor": "National Institute of Informatics", "subitem_conference_sponsor_language": "en"}], "subitem_conference_venues": [{"subitem_conference_venue": "National Institute of Informatics", "subitem_conference_venue_language": "en"}]}]}, "item_30002_creator2": {"attribute_name": "作成者", "attribute_type": "creator", "attribute_value_mlt": [{"creatorNames": [{"creatorName": "Tetsuya Sakai", "creatorNameLang": "en"}]}, {"creatorNames": [{"creatorNameLang": "en"}]}]}, "item_30002_date11": {"attribute_name": "日付", "attribute_value_mlt": [{"subitem_date_issued_datetime": "2023-12-12", "subitem_date_issued_type": "Issued"}, {"subitem_date_issued_datetime": "2023-09-12", "subitem_date_issued_type": "Updated"}]}, "item_30002_description9": {"attribute_name": "内容記述", "attribute_value_mlt": [{"subitem_description": "NTCIR-17 introduced the FairWeb-1 task,which evaluated web page rankingsin terms of both relevance and group fairness.The present study shows how their evaluation frameworkcan be extended for the evaluation ofmulti-turn, textual conversational search systems.By using the full test topic set of FairWeb-1to harvest actual user-system conversations from the New Bing and Google Bard,we demonstrate how a series of system turns can be evaluatedusing our evaluation framework,which we callGFRC (Group Fairness and Relevance of Conversations).In addition, based onobservations from our pilot experiment,we briefly discuss a few open questions in human-in-the-loopevaluation of conversational search in general.", "subitem_description_language": "en", "subitem_description_type": "Abstract"}]}, "item_30002_file35": {"attribute_name": "ファイル情報", "attribute_type": "file", "attribute_value_mlt": [{"accessrole": "open_access", "date": [{"dateType": "Available", "dateValue": "2023-12-11"}], "download_preview_message": "", "file_order": 0, "filename": "01-EVIA2023-EVIA-SakaiT.pdf", "filesize": [{"value": "1.7 MB"}], "format": "application/pdf", "future_date_message": "", "is_thumbnail": false, "mimetype": "application/pdf", "size": 1700000.0, "url": {"url": "https://repository.nii.ac.jp/record/2001350/files/01-EVIA2023-EVIA-SakaiT.pdf"}, "version_id": "fbec66be-38b4-443e-9397-dacb228504fb"}]}, "item_30002_identifier_registration17": {"attribute_name": "ID登録", "attribute_value_mlt": [{"subitem_identifier_reg_text": "10.20736/0002001350", "subitem_identifier_reg_type": "JaLC"}]}, "item_30002_language12": {"attribute_name": "言語", "attribute_value_mlt": [{"subitem_language": "eng"}]}, "item_30002_page_start27": {"attribute_name": "開始ページ", "attribute_value_mlt": [{"subitem_start_page": "none"}]}, "item_30002_publisher10": {"attribute_name": "出版者", "attribute_value_mlt": [{"subitem_publisher": "NII Institutional Repository", "subitem_publisher_language": "en"}]}, "item_30002_relation18": {"attribute_name": "関連情報", "attribute_value_mlt": [{"subitem_relation_name": [{"subitem_relation_name_language": "en", "subitem_relation_name_text": "NTCIR-17 Conference"}], "subitem_relation_type": "isReferencedBy", "subitem_relation_type_id": {"subitem_relation_type_id_text": "https://research.nii.ac.jp/ntcir/ntcir-17/index.html", "subitem_relation_type_select": "URI"}}]}, "item_30002_resource_type13": {"attribute_name": "資源タイプ", "attribute_value_mlt": [{"resourcetype": "conference paper", "resourceuri": "http://purl.org/coar/resource_type/c_5794"}]}, "item_30002_subject8": {"attribute_name": "主題", "attribute_value_mlt": [{"subitem_subject": "conversational search", "subitem_subject_language": "en", "subitem_subject_scheme": "Other"}, {"subitem_subject": "dialogue systems", "subitem_subject_language": "en", "subitem_subject_scheme": "Other"}, {"subitem_subject": "evaluation", "subitem_subject_language": "en", "subitem_subject_scheme": "Other"}, {"subitem_subject": "fair ranking", "subitem_subject_language": "en", "subitem_subject_scheme": "Other"}, {"subitem_subject": "group fairness", "subitem_subject_language": "en", "subitem_subject_scheme": "Other"}, {"subitem_subject": "NTCIR", "subitem_subject_language": "en", "subitem_subject_scheme": "Other"}, {"subitem_subject": "web search", "subitem_subject_language": "en", "subitem_subject_scheme": "Other"}]}, "item_30002_title0": {"attribute_name": "タイトル", "attribute_value_mlt": [{"subitem_title": "Fairness-based Evaluation of Conversational Search: A Pilot Study", "subitem_title_language": "en"}]}, "item_title": "Fairness-based Evaluation of Conversational Search: A Pilot Study", "item_type_id": "30002", "owner": "23", "path": ["1697763112205"], "permalink_uri": "https://doi.org/10.20736/0002001350", "pubdate": {"attribute_name": "公開日", "attribute_value": "2023-12-12"}, "publish_date": "2023-12-12", "publish_status": "0", "recid": "2001350", "relation": {}, "relation_version_is_last": true, "title": ["Fairness-based Evaluation of Conversational Search: A Pilot Study"], "weko_shared_id": -1}
Fairness-based Evaluation of Conversational Search: A Pilot Study
https://doi.org/10.20736/0002001350
https://doi.org/10.20736/00020013500001f7e9-9796-4503-8612-6726974162fe
名前 / ファイル | ライセンス | アクション |
---|---|---|
01-EVIA2023-EVIA-SakaiT.pdf (1.7 MB)
|
|
Item type | デフォルトアイテムタイプ(フル)(1) | |||||||
---|---|---|---|---|---|---|---|---|
公開日 | 2023-12-12 | |||||||
タイトル | ||||||||
言語 | en | |||||||
タイトル | Fairness-based Evaluation of Conversational Search: A Pilot Study | |||||||
作成者 |
Tetsuya Sakai
× Tetsuya Sakai
|
|||||||
主題 | ||||||||
言語 | en | |||||||
主題Scheme | Other | |||||||
主題 | conversational search | |||||||
主題 | ||||||||
言語 | en | |||||||
主題Scheme | Other | |||||||
主題 | dialogue systems | |||||||
主題 | ||||||||
言語 | en | |||||||
主題Scheme | Other | |||||||
主題 | evaluation | |||||||
主題 | ||||||||
言語 | en | |||||||
主題Scheme | Other | |||||||
主題 | fair ranking | |||||||
主題 | ||||||||
言語 | en | |||||||
主題Scheme | Other | |||||||
主題 | group fairness | |||||||
主題 | ||||||||
言語 | en | |||||||
主題Scheme | Other | |||||||
主題 | NTCIR | |||||||
主題 | ||||||||
言語 | en | |||||||
主題Scheme | Other | |||||||
主題 | web search | |||||||
内容記述 | ||||||||
内容記述タイプ | Abstract | |||||||
内容記述 | NTCIR-17 introduced the FairWeb-1 task,which evaluated web page rankingsin terms of both relevance and group fairness.The present study shows how their evaluation frameworkcan be extended for the evaluation ofmulti-turn, textual conversational search systems.By using the full test topic set of FairWeb-1to harvest actual user-system conversations from the New Bing and Google Bard,we demonstrate how a series of system turns can be evaluatedusing our evaluation framework,which we callGFRC (Group Fairness and Relevance of Conversations).In addition, based onobservations from our pilot experiment,we briefly discuss a few open questions in human-in-the-loopevaluation of conversational search in general. | |||||||
言語 | en | |||||||
出版者 | ||||||||
言語 | en | |||||||
出版者 | NII Institutional Repository | |||||||
日付 | ||||||||
日付 | 2023-12-12 | |||||||
日付タイプ | Issued | |||||||
日付 | ||||||||
日付 | 2023-09-12 | |||||||
日付タイプ | Updated | |||||||
言語 | ||||||||
言語 | eng | |||||||
資源タイプ | ||||||||
資源タイプ識別子 | http://purl.org/coar/resource_type/c_5794 | |||||||
資源タイプ | conference paper | |||||||
ID登録 | ||||||||
ID登録 | 10.20736/0002001350 | |||||||
ID登録タイプ | JaLC | |||||||
関連情報 | ||||||||
関連タイプ | isReferencedBy | |||||||
識別子タイプ | URI | |||||||
関連識別子 | https://research.nii.ac.jp/ntcir/ntcir-17/index.html | |||||||
言語 | en | |||||||
関連名称 | NTCIR-17 Conference | |||||||
開始ページ | ||||||||
開始ページ | none | |||||||
会議記述 | ||||||||
会議名 | NTCIR-17 Conference | |||||||
言語 | en | |||||||
回次 | 17 | |||||||
主催機関 | National Institute of Informatics | |||||||
言語 | en | |||||||
開始年 | 2023 | |||||||
開始月 | 12 | |||||||
開始日 | 12 | |||||||
終了年 | 2023 | |||||||
終了月 | 12 | |||||||
終了日 | 15 | |||||||
開催期間 | December 12-15, 2023 | |||||||
言語 | en | |||||||
開催会場 | National Institute of Informatics | |||||||
言語 | en | |||||||
開催国 | JPN |