My task was to provide my web search engine, which I had already implemented in a previous task, on a virtual machine.
I also had to fulfill the following requirements:
Set the starting page of your search engine to index.html so that one can access it on:http://virtual_machine_IP:8080/is-project/index.html
Your JSON interface should have the following URL:http://virtual_machine_IP:8080/is-project/json?query=keywords&k=K&score=Swhere keywords are the query keywords separated by a “+” sign, K is the number of the documents retrieved, and S ∈ {1, 2, 3} is the scoring function where 1 is TF*IDF, 2 is BM25, and 3 is the combined score. The language should be fixed to English.For example, if we want to get the JSON document for the query “database course”, where we want to retrieve the first 10 results using the combined score, the URL should be:http://virtual_machine_IP:8080/is-project/json?query=database+course&k=10&score=3
My problem is that when I call up the first URL, the HTML user interface of the index.html file is not displayed, but the following:
{“error”: “Query parameter is missing or empty”, “status”: “400”}
If I use the URL http://131.246.117.34:8080/is-project/index.html?q=database%20course&conjunctive=false&k=5&scoringModel=combined&language=auto instead, I get a list of results, but in JSON format and not in the HTML user interface.
Here is the code I am using for SearchServlet and index.html:
import com.google.gson.Gson;import com.google.gson.GsonBuilder;import jakarta.servlet.ServletException;import jakarta.servlet.annotation.WebServlet;import jakarta.servlet.http.HttpServlet;import jakarta.servlet.http.HttpServletRequest;import jakarta.servlet.http.HttpServletResponse;import java.io.IOException;import java.io.PrintWriter;import java.sql.SQLException;import java.util.*;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.atomic.AtomicInteger;import java.util.logging.Level;import java.util.logging.Logger;import java.util.regex.Matcher;import java.util.regex.Pattern;/** * Servlet that handles search requests for the web search engine. * Maps to the "/search" URL pattern. */@WebServlet("/search")public class SearchServlet extends HttpServlet { private SearchEngine searchEngine; private SpellChecker spellChecker; private LanguageDetector languageDetector; private static final int DEFAULT_RESULTS_LIMIT = 20; private static final Logger LOGGER = Logger.getLogger(SearchServlet.class.getName()); private RateLimiter rateLimiter; /** * Initializes the servlet by creating a new SearchEngine instance. * @throws ServletException if there's an error initializing the SearchEngine */ @Override public void init() throws ServletException { // Load the PostgreSQL JDBC driver // This step is necessary to ensure the driver is available for database connections try { Class.forName("org.postgresql.Driver"); searchEngine = new SearchEngine(); spellChecker = new SpellChecker(); languageDetector = new LanguageDetector(); rateLimiter = new RateLimiter(); LOGGER.info("SearchEngine, SpellChecker, LanguageDetector, and RateLimiter initialized successfully"); } catch (SQLException | ClassNotFoundException e) { LOGGER.log(Level.SEVERE, "Error initializing SearchEngine, SpellChecker, or LanguageDetector", e); ErrorLogger.logError("Error initializing components", e); throw new ServletException("Error initializing components", e); } } /** * Handles GET requests for search operations. * Processes the query, performs the search, and returns results as JSON. * @param request the HttpServletRequest object * @param response the HttpServletResponse object * @throws ServletException if there's a servlet-specific error * @throws IOException if an I/O error occurs */ @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { String query = request.getParameter("q"); if (query == null || query.trim().isEmpty()) { sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Query parameter is missing or empty"); return; } String ipAddress = request.getRemoteAddr(); if (!rateLimiter.allowRequest(ipAddress)) { response.setStatus(429); response.getWriter().write("Rate limit exceeded"); ErrorLogger.logError("Rate limit exceeded for IP: " + ipAddress, null); return; } boolean isConjunctive = Boolean.parseBoolean(request.getParameter("conjunctive")); String scoringModel = request.getParameter("scoringModel"); int k = parseResultsLimit(request.getParameter("k")); String preferredLanguage = request.getParameter("language"); String detectedLanguage = "default"; boolean languageDetected = false; LOGGER.info("Received search request: query=" + query +", isConjunctive=" + isConjunctive +", scoringModel=" + scoringModel +", k=" + k +", language=" + preferredLanguage); try { if (query == null || query.trim().isEmpty()) { throw new IllegalArgumentException("Query parameter is missing or empty"); } List<String> terms = parseQuery(query); String siteFilter = extractSiteFilter(terms); List<String> exactPhrases = extractExactPhrases(terms); String languageForSpellCheck = preferredLanguage; if ("auto".equals(preferredLanguage)) { detectedLanguage = languageDetector.detectLanguage(String.join(" ", terms)); languageForSpellCheck = detectedLanguage; languageDetected = true; LOGGER.info("Auto-detected language: " + detectedLanguage); } Map<String, List<String>> spellingSuggestions = checkSpelling(terms, languageForSpellCheck); LOGGER.info("Spelling suggestions: " + spellingSuggestions); List<SearchEngine.SearchResult> results; if (exactPhrases.isEmpty()) { results = searchEngine.search(terms, k, isConjunctive, scoringModel, languageForSpellCheck); } else { results = searchEngine.searchWithExactPhrases(terms, exactPhrases, k, isConjunctive, scoringModel, languageForSpellCheck); } if (results != null && !results.isEmpty() && "auto".equals(preferredLanguage)) { detectedLanguage = results.get(0).language; } results = filterResults(results, siteFilter); Map<String, Object> responseData = new LinkedHashMap<>(); List<Map<String, Object>> formattedResults = new ArrayList<>(); for (int i = 0; i < results.size(); i++) { SearchEngine.SearchResult result = results.get(i); Map<String, Object> formattedResult = new LinkedHashMap<>(); formattedResult.put("rank", i + 1); formattedResult.put("url", result.url); formattedResult.put("score", result.score); formattedResult.put("language", result.language); formattedResults.add(formattedResult); } responseData.put("resultList", formattedResults); Map<String, Object> queryInfo = new LinkedHashMap<>(); queryInfo.put("k", k); queryInfo.put("query", query); queryInfo.put("scoringModel", scoringModel); queryInfo.put("language", preferredLanguage); queryInfo.put("detectedLanguage", detectedLanguage); queryInfo.put("languageDetected", languageDetected); responseData.put("query", queryInfo); List<Map<String, Object>> stats = calculateStatistics(terms, exactPhrases, languageForSpellCheck); responseData.put("stat", stats); int cw = searchEngine.getCollectionWordCount(); responseData.put("cw", cw); responseData.put("spellingSuggestions", spellingSuggestions); if (results.isEmpty() && !spellingSuggestions.isEmpty()) { List<String> alternativeQueries = generateAlternativeQueries(terms, spellingSuggestions); responseData.put("alternativeQueries", alternativeQueries); } response.setContentType("application/json"); response.setCharacterEncoding("UTF-8"); Gson gson = new GsonBuilder().setPrettyPrinting().create(); String jsonResponse = gson.toJson(responseData); response.getWriter().write(jsonResponse); LOGGER.info("Response sent successfully"); } catch (SQLException e) { LOGGER.log(Level.SEVERE, "SQL error occurred during search", e); ErrorLogger.logError("SQL error occurred during search", e); sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Database error occurred"); } catch (IllegalArgumentException e) { LOGGER.log(Level.WARNING, "Invalid input parameter", e); ErrorLogger.logError("Invalid input parameter: " + e.getMessage(), e); sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, e.getMessage()); } catch (Exception e) { LOGGER.log(Level.SEVERE, "Unexpected error occurred during search", e); ErrorLogger.logError("Unexpected error occurred during search", e); sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "An unexpected error occurred"); } } /** * Parses the query string into a list of terms, preserving quoted phrases. * @param query the input query string * @return a list of parsed terms */ private List<String> parseQuery(String query) { List<String> terms = new ArrayList<>(); Pattern pattern = Pattern.compile("\"([^\"]*)\"|(\\S+)"); Matcher matcher = pattern.matcher(query); while (matcher.find()) { if (matcher.group(1) != null) { terms.add("\"" + matcher.group(1) +"\""); } else { terms.add(matcher.group(2)); } } return terms; } /** * Extracts and removes the site filter from the list of terms. * @param terms the list of query terms * @return the extracted site filter, or null if not present */ private String extractSiteFilter(List<String> terms) { for (Iterator<String> it = terms.iterator(); it.hasNext();) { String term = it.next(); if (term.startsWith("site:")) { it.remove(); return term.substring(5); } } return null; } /** * Extracts and removes exact phrases (quoted terms) from the list of terms. * @param terms the list of query terms * @return a list of extracted exact phrases */ private List<String> extractExactPhrases(List<String> terms) { List<String> exactPhrases = new ArrayList<>(); for (Iterator<String> it = terms.iterator(); it.hasNext();) { String term = it.next(); if (term.startsWith("\"") && term.endsWith("\"")) { it.remove(); exactPhrases.add(term.substring(1, term.length() - 1)); } } return exactPhrases; } /** * Filters search results based on the site filter. * @param results the list of search results * @param siteFilter the site filter to apply * @param exactPhrases the list of exact phrases (unused in this method, but could be used for additional filtering) * @return the filtered list of search results */ private List<SearchEngine.SearchResult> filterResults(List<SearchEngine.SearchResult> results, String siteFilter) { if (siteFilter == null) { return results; } List<SearchEngine.SearchResult> filteredResults = new ArrayList<>(); for (SearchEngine.SearchResult result : results) { if (result.url.contains(siteFilter)) { filteredResults.add(result); } } return filteredResults; } /** * Calculates statistics for the search terms and exact phrases. * @param terms the list of search terms * @param exactPhrases the list of exact phrases * @return a list of maps containing term statistics * @throws SQLException if there's an error accessing the database */ private List<Map<String, Object>> calculateStatistics(List<String> terms, List<String> exactPhrases, String language) throws SQLException { Map<String, Integer> termFrequencies = searchEngine.getTermFrequencies(terms, exactPhrases, language); List<Map<String, Object>> stats = new ArrayList<>(); for (String term : terms) { Map<String, Object> termStat = new LinkedHashMap<>(); termStat.put("term", term); termStat.put("df", termFrequencies.getOrDefault(spellChecker.stemTerm(term, language), 0)); stats.add(termStat); } for (String phrase : exactPhrases) { Map<String, Object> phraseStat = new LinkedHashMap<>(); phraseStat.put("term", phrase); int minDf = Arrays.stream(phrase.split("\\s+")) .mapToInt(term -> termFrequencies.getOrDefault(spellChecker.stemTerm(term, language), Integer.MAX_VALUE)) .min() .orElse(0); phraseStat.put("df", minDf); stats.add(phraseStat); } return stats; } /** * Parses the results limit parameter. * @param kParam the k parameter from the request * @return the parsed limit, or the default limit if parsing fails */ private int parseResultsLimit(String kParam) { if (kParam == null || kParam.isEmpty()) { return DEFAULT_RESULTS_LIMIT; } try { int k = Integer.parseInt(kParam); return k > 0 ? k : DEFAULT_RESULTS_LIMIT; } catch (NumberFormatException e) { return DEFAULT_RESULTS_LIMIT; } } private void sendErrorResponse(HttpServletResponse response, int statusCode, String message) throws IOException { response.setStatus(statusCode); response.setContentType("application/json"); response.setCharacterEncoding("UTF-8"); Map<String, String> errorResponse = new HashMap<>(); errorResponse.put("error", message); errorResponse.put("status", String.valueOf(statusCode)); new Gson().toJson(errorResponse, response.getWriter()); } private Map<String, List<String>> checkSpelling(List<String> terms, String language) { try { return spellChecker.checkSpellingForQuery(terms, language); } catch (SQLException e) { LOGGER.log(Level.SEVERE, "Error checking spelling", e); return Collections.emptyMap(); } } private List<String> generateAlternativeQueries(List<String> originalTerms, Map<String, List<String>> spellingSuggestions) { List<String> alternativeQueries = new ArrayList<>(); for (int i = 0; i < 5; i++) { List<String> alternativeTerms = new ArrayList<>(originalTerms); boolean changed = false; for (Map.Entry<String, List<String>> entry : spellingSuggestions.entrySet()) { String originalTerm = entry.getKey(); List<String> suggestions = entry.getValue(); if (!suggestions.isEmpty()) { int index = alternativeTerms.indexOf(originalTerm); if (index != -1) { alternativeTerms.set(index, suggestions.get(i % suggestions.size())); changed = true; } } } if (changed) { alternativeQueries.add(String.join(" ", alternativeTerms)); } else { break; } } return alternativeQueries; } @Override public void destroy() { try { if (spellChecker != null) { spellChecker.close(); } } catch (SQLException e) { LOGGER.log(Level.SEVERE, "Error closing SpellChecker", e); ErrorLogger.logError("Error closing SpellChecker", e); } } private static class RateLimiter { private static final int GLOBAL_LIMIT = 10; // 10 requests per second globally private static final int IP_LIMIT = 1; // 1 request per second per IP private final ConcurrentHashMap<String, AtomicInteger> ipRequests = new ConcurrentHashMap<>(); private final AtomicInteger globalRequests = new AtomicInteger(0); private long lastResetTime = System.currentTimeMillis(); public boolean allowRequest(String ipAddress) { long currentTime = System.currentTimeMillis(); if (currentTime - lastResetTime > 1000) { ipRequests.clear(); globalRequests.set(0); lastResetTime = currentTime; } if (globalRequests.incrementAndGet() > GLOBAL_LIMIT) { return false; } AtomicInteger ipCounter = ipRequests.computeIfAbsent(ipAddress, k -> new AtomicInteger(0)); return ipCounter.incrementAndGet() <= IP_LIMIT; } } @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { doGet(request, response); }}Unfortunately i can't upload the full code of the index.html so here is a snippet:
<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>WebSearchEngine</title><!-- External CSS for icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css"><style> /* CSS styles for the page layout and components */</style></head><body><!-- Main container for the search engine interface --><div class="container"><!-- Logo container --><div class="logo-container"><img src="Logo.png" alt="WebSearchEngine Logo" class="logo"></div><h1>WebSearchEngine</h1><!-- Search form --><form id="searchForm" action="/is-project/index.html" method="get"><!-- Search input field --><input type="text" id="searchInput" placeholder="Enter search terms..." required><!-- Input for number of results to display --><input type="number" id="kInput" placeholder="k" min="1" value="20"><!-- Scoring model selection --><select id="scoringModel"><option value="tfidf">TF*IDF</option><option value="bm25">BM25</option><option value="combined">Combined</option></select><!-- Language selection dropdown --><select id="languageSelect"><option value="auto">Auto Detect</option><option value="english">English</option><option value="german">German</option></select><!-- Search button with icon --><button type="submit" id="searchButton"><i class="fas fa-search"></i> Search</button></form><!-- Search options --><div id="searchOptions"><label><input type="checkbox" id="conjunctiveSearch"> Conjunctive Search</label></div><!-- Loading indicator --><div id="loading">Loading results...</div><!-- Container for search results --><ul id="results"></ul><div id="spellingSuggestions"></div><div id="alternativeQueries"></div><!-- Container for parsed query information --><div id="parsedQuery"></div><!-- Container for search statistics --><div id="statistics"></div></div><script> // Event listener for form submission document.getElementById('searchForm').addEventListener('submit', function(e) { e.preventDefault(); const query = document.getElementById('searchInput').value; const isConjunctive = document.getElementById('conjunctiveSearch').checked; const k = document.getElementById('kInput').value; const scoringModel = document.getElementById('scoringModel').value; const language = document.getElementById('languageSelect').value; search(query, isConjunctive, k, scoringModel, language); }); // Function to perform the search async function search(query, isConjunctive, k, scoringModel, language) { const loadingElement = document.getElementById('loading'); const resultsElement = document.getElementById('results'); loadingElement.style.display = 'block'; resultsElement.innerHTML = ''; try { const response = await fetch(`/is-project/index.html?q=${encodeURIComponent(query)}&conjunctive=${isConjunctive}&k=${k}&scoringModel=${scoringModel}&language=${language}`); const contentType = response.headers.get("content-type"); if (contentType && contentType.indexOf("application/json") !== -1) { const data = await response.json(); if (!response.ok) { throw new Error(data.error || `HTTP error! status: ${response.status}`); } displayResults(data); displayParsedQuery(query, k, scoringModel, data.query.language, data.query.languageDetected, data.query.detectedLanguage); displayStatistics(data.stat, data.cw); } else { // If the response is not JSON, it's probably the HTML page resultsElement.innerHTML = await response.text(); } } catch (error) { console.error('Error during search:', error); resultsElement.innerHTML = `<li class="error">An error occurred during the search: ${error.message}</li>`; logErrorToServer(error); } finally { loadingElement.style.display = 'none'; } } // Function to log errors to the server async function logErrorToServer(error) { try { await fetch('/is-project/log-error', { method: 'POST', headers: {'Content-Type': 'application/json', }, body: JSON.stringify({ message: error.message, stack: error.stack, timestamp: new Date().toISOString() }) }); } catch (logError) { console.error('Failed to log error to server:', logError); } } // Function to display search results function displayResults(data) { const resultsElement = document.getElementById('results'); const spellingSuggestionsElement = document.getElementById('spellingSuggestions'); const alternativeQueriesElement = document.getElementById('alternativeQueries'); resultsElement.innerHTML = ''; spellingSuggestionsElement.innerHTML = ''; alternativeQueriesElement.innerHTML = ''; if (data.resultList.length === 0) { resultsElement.innerHTML = '<li>No results found.</li>'; } else { data.resultList.forEach((result, index) => { const li = document.createElement('li'); const fullUrl = result.url.startsWith('http') ? result.url : `http://${result.url}`; li.innerHTML = `<div>${index + 1}. <a href="${fullUrl}" target="_blank" rel="noopener noreferrer">${result.url}</a></div><div class="score">Score: ${result.score.toFixed(4)}, Language: ${result.language}</div> `; resultsElement.appendChild(li); }); } // Display spelling suggestions if (data.spellingSuggestions && Object.keys(data.spellingSuggestions).length > 0) { let suggestionsHtml = '<h3>Spelling Suggestions:</h3><ul>'; for (const [term, suggestions] of Object.entries(data.spellingSuggestions)) { suggestionsHtml += `<li>${term}: ${suggestions.join(', ')}</li>`; } suggestionsHtml += '</ul>'; spellingSuggestionsElement.innerHTML = suggestionsHtml; } // Display alternative queries if (data.alternativeQueries && data.alternativeQueries.length > 0) { let alternativeQueriesHtml = '<h3>Did you mean:</h3><ul>'; data.alternativeQueries.forEach(query => { alternativeQueriesHtml += `<li><a href="#" onclick="document.getElementById('searchInput').value='${query}'; search('${query}', false, 20, 'tfidf', 'auto'); return false;">${query}</a></li>`; }); alternativeQueriesHtml += '</ul>'; alternativeQueriesElement.innerHTML = alternativeQueriesHtml; } } // Function to display parsed query information function displayParsedQuery(query, k, scoringModel, preferredLanguage, languageDetected, detectedLanguage) { const parsedQueryElement = document.getElementById('parsedQuery'); const parts = parseQuery(query); let html = '<h3>Query Information:</h3><ul>'; html += `<li><strong>k: </strong>${k}</li>`; html += `<li><strong>query: </strong>${query}</li>`; html += `<li><strong>scoring model: </strong>${scoringModel}</li>`; html += `<li><strong>preferred language: </strong>${preferredLanguage}</li>`; html += `<li><strong>language detected: </strong>${languageDetected ? 'Yes' : 'No'}</li>`; if (languageDetected) { html += `<li><strong>detected language: </strong>${detectedLanguage}</li>`; } parts.forEach(part => { html += `<li><strong>${part.type}:</strong> ${part.value}</li>`; }); html += '</ul>'; parsedQueryElement.innerHTML = html; } // Function to display search statistics function displayStatistics(stats, cw) { const statisticsElement = document.getElementById('statistics'); let html = '<h3>Statistics:</h3><ul>'; stats.forEach(stat => { html += `<li><strong>${stat.term}:</strong> df = ${stat.df}</li>`; }); html += `<li><strong>Total terms in collection:</strong> ${cw}</li>`; html += '</ul>'; statisticsElement.innerHTML = html; } // Function to parse the query into different parts (site, exact phrase, or term) function parseQuery(query) { const parts = query.match(/(?:[^\s"]+|"[^"]*")+/g) || []; return parts.map(part => { if (part.startsWith('site:')) { return { type: 'Site', value: part.slice(5) }; } else if (part.startsWith('"') && part.endsWith('"')) { return { type: 'Exact', value: part.slice(1, -1) }; } else { return { type: 'Term', value: part }; } }); }</script></body></html>Could someone please help me and explain what my error is?
The virtual machine is running ubuntu and the tomcat version is Tomcat 11